@diottodev/runic 1.0.0 → 1.1.0

package/engineering/skills/interview-system-designer/references/debrief_facilitation_guide.md

@@ -0,0 +1,319 @@
+# Interview Debrief Facilitation Guide
+
+This guide provides a comprehensive framework for conducting effective, unbiased interview debriefs that lead to consistent hiring decisions. Use this to facilitate productive discussions that focus on evidence-based evaluation.
+
+## Pre-Debrief Preparation
+
+### Facilitator Responsibilities
+- [ ] **Review all interviewer feedback** before the meeting
+- [ ] **Identify significant score discrepancies** that need discussion
+- [ ] **Prepare discussion agenda** with time allocations
+- [ ] **Gather role requirements** and competency framework
+- [ ] **Review any flags or special considerations** noted during interviews
+- [ ] **Ensure all required materials** are available (scorecards, rubrics, candidate resume)
+- [ ] **Set up meeting logistics** (room, video conference, screen sharing)
+- [ ] **Send agenda to participants** 30 minutes before meeting
+
+### Required Materials Checklist
+- [ ] Candidate resume and application materials
+- [ ] Job description and competency requirements
+- [ ] Individual interviewer scorecards
+- [ ] Scoring rubrics and competency definitions
+- [ ] Interview notes and documentation
+- [ ] Any technical assessments or work samples
+- [ ] Company hiring standards and calibration examples
+- [ ] Bias mitigation reminders and prompts
+
+### Participant Preparation Requirements
+- [ ] All interviewers must **complete independent scoring** before debrief
+- [ ] **Submit written feedback** with specific evidence for each competency
+- [ ] **Review scoring rubrics** to ensure consistent interpretation
+- [ ] **Prepare specific examples** to support scoring decisions
+- [ ] **Flag any concerns or unusual circumstances** that affected assessment
+- [ ] **Avoid discussing candidate** with other interviewers before debrief
+- [ ] **Come prepared to defend scores** with concrete evidence
+- [ ] **Be ready to adjust scores** based on additional evidence shared
+
+## Debrief Meeting Structure
+
+### Opening (5 minutes)
+1. **State meeting purpose**: Make hiring decision based on evidence
+2. **Review agenda and time limits**: Keep discussion focused and productive
+3. **Remind of bias mitigation principles**: Focus on competencies, not personality
+4. **Confirm confidentiality**: Discussion stays within hiring team
+5. **Establish ground rules**: One person speaks at a time, evidence-based discussion
+
+### Individual Score Sharing (10-15 minutes)
+- **Go around the room systematically** - each interviewer shares scores independently
+- **No discussion or challenges yet** - just data collection
+- **Record scores on shared document** visible to all participants
+- **Note any abstentions** or "insufficient data" responses
+- **Identify clear patterns** and discrepancies without commentary
+- **Flag any scores requiring explanation** (1s or 4s typically need strong evidence)
+
+### Competency-by-Competency Discussion (30-40 minutes)
+
+#### For Each Core Competency:
+
+**1. Present Score Distribution (2 minutes)**
+- Display all scores for this competency
+- Note range and any outliers
+- Identify if consensus exists or discussion needed
+
+**2. Evidence Sharing (5-8 minutes per competency)**
+- Start with interviewers who assessed this competency directly
+- Share specific examples and observations
+- Focus on what candidate said/did, not interpretations
+- Allow questions for clarification (not challenges yet)
+
+**3. Discussion and Calibration (3-5 minutes)**
+- Address significant discrepancies (>1 point difference)
+- Challenge vague or potentially biased language
+- Seek additional evidence if needed
+- Allow score adjustments based on new information
+- Reach consensus or note dissenting views
+
+#### Structured Discussion Questions:
+- **"What specific evidence supports this score?"**
+- **"Can you provide the exact example or quote?"**
+- **"How does this compare to our rubric definition?"**
+- **"Would this response receive the same score regardless of who gave it?"**
+- **"Are we evaluating the competency or making assumptions?"**
+- **"What would need to change for this to be the next level up/down?"**
+
+### Overall Recommendation Discussion (10-15 minutes)
+
+#### Weighted Score Calculation
+1. **Apply competency weights** based on role requirements
+2. **Calculate overall weighted average**
+3. **Check minimum threshold requirements**
+4. **Consider any veto criteria** (critical competency failures)
+
+#### Final Recommendation Options
+- **Strong Hire**: Exceeds requirements in most areas, clear value-add
+- **Hire**: Meets requirements with growth potential
+- **No Hire**: Doesn't meet minimum requirements for success
+- **Strong No Hire**: Significant gaps that would impact team/company
+
+#### Decision Rationale Documentation
+- **Summarize key strengths** with specific evidence
+- **Identify development areas** with specific examples
+- **Explain final recommendation** with competency-based reasoning
+- **Note any dissenting opinions** and reasoning
+- **Document onboarding considerations** if hiring
+
+### Closing and Next Steps (5 minutes)
+- **Confirm final decision** and documentation
+- **Assign follow-up actions** (feedback delivery, offer preparation, etc.)
+- **Schedule any additional interviews** if needed
+- **Review timeline** for candidate communication
+- **Remind confidentiality** of discussion and decision
+
+## Facilitation Best Practices
+
+### Creating Psychological Safety
+- **Encourage honest feedback** without fear of judgment
+- **Validate different perspectives** and assessment approaches  
+- **Address power dynamics** - ensure junior voices are heard
+- **Model vulnerability** - admit when evidence changes your mind
+- **Focus on learning** and calibration, not winning arguments
+- **Thank participants** for thorough preparation and thoughtful input
+
+### Managing Difficult Conversations
+
+#### When Scores Vary Significantly
+1. **Acknowledge the discrepancy** without judgment
+2. **Ask for specific evidence** from each scorer
+3. **Look for different interpretations** of the same data
+4. **Consider if different questions** revealed different competency levels
+5. **Check for bias patterns** in reasoning
+6. **Allow time for reflection** and potential score adjustments
+
+#### When Someone Uses Biased Language
+1. **Pause the conversation** gently but firmly
+2. **Ask for specific evidence** behind the assessment
+3. **Reframe in competency terms** - "What specific skills did this demonstrate?"
+4. **Challenge assumptions** - "Help me understand how we know that"
+5. **Redirect to rubric** - "How does this align with our scoring criteria?"
+6. **Document and follow up** privately if bias persists
+
+#### When the Discussion Gets Off Track
+- **Redirect to competencies**: "Let's focus on the technical skills demonstrated"
+- **Ask for evidence**: "What specific example supports that assessment?"  
+- **Reference rubrics**: "How does this align with our level 3 definition?"
+- **Manage time**: "We have 5 minutes left on this competency"
+- **Table unrelated issues**: "That's important but separate from this hire decision"
+
+### Encouraging Evidence-Based Discussion
+
+#### Good Evidence Examples
+- **Direct quotes**: "When asked about debugging, they said..."
+- **Specific behaviors**: "They organized their approach by first..."
+- **Observable outcomes**: "Their code compiled on first run and handled edge cases"
+- **Process descriptions**: "They walked through their problem-solving step by step"
+- **Measurable results**: "They identified 3 optimization opportunities"
+
+#### Poor Evidence Examples
+- **Gut feelings**: "They just seemed off"
+- **Comparisons**: "Not as strong as our last hire"
+- **Assumptions**: "Probably wouldn't fit our culture"
+- **Vague impressions**: "Didn't seem passionate"
+- **Irrelevant factors**: "Their background is different from ours"
+
+### Managing Group Dynamics
+
+#### Ensuring Equal Participation
+- **Direct questions** to quieter participants
+- **Prevent interrupting** and ensure everyone finishes thoughts
+- **Balance speaking time** across all interviewers
+- **Validate minority opinions** even if not adopted
+- **Check for unheard perspectives** before finalizing decisions
+
+#### Handling Strong Personalities
+- **Set time limits** for individual speaking
+- **Redirect monopolizers**: "Let's hear from others on this"
+- **Challenge confidently stated opinions** that lack evidence
+- **Support less assertive voices** in expressing dissenting views
+- **Focus on data**, not personality or seniority in decision making
+
+## Bias Interruption Strategies
+
+### Affinity Bias Interruption
+- **Notice pattern**: Positive assessment seems based on shared background/interests
+- **Interrupt with**: "Let's focus on the job-relevant skills they demonstrated"
+- **Redirect to**: Specific competency evidence and measurable outcomes
+- **Document**: Note if personal connection affected professional assessment
+
+### Halo/Horn Effect Interruption
+- **Notice pattern**: One area strongly influencing assessment of unrelated areas
+- **Interrupt with**: "Let's score each competency independently"
+- **Redirect to**: Specific evidence for each individual competency area
+- **Recalibrate**: Ask for separate examples supporting each score
+
+### Confirmation Bias Interruption
+- **Notice pattern**: Only seeking/discussing evidence that supports initial impression
+- **Interrupt with**: "What evidence might suggest a different assessment?"
+- **Redirect to**: Consider alternative interpretations of the same data
+- **Challenge**: "How might we be wrong about this assessment?"
+
+### Attribution Bias Interruption
+- **Notice pattern**: Attributing success to luck/help for some demographics, skill for others
+- **Interrupt with**: "What role did the candidate play in achieving this outcome?"
+- **Redirect to**: Candidate's specific contributions and decision-making
+- **Standardize**: Apply same attribution standards across all candidates
+
+## Decision Documentation Framework
+
+### Required Documentation Elements
+1. **Final scores** for each assessed competency
+2. **Overall recommendation** with supporting rationale
+3. **Key strengths** with specific evidence
+4. **Development areas** with specific examples
+5. **Dissenting opinions** if any, with reasoning
+6. **Special considerations** or accommodation needs
+7. **Next steps** and timeline for decision communication
+
+### Evidence Quality Standards
+- **Specific and observable**: What exactly did the candidate do or say?
+- **Job-relevant**: How does this relate to success in the role?
+- **Measurable**: Can this be quantified or clearly described?
+- **Unbiased**: Would this evidence be interpreted the same way regardless of candidate demographics?
+- **Complete**: Does this represent the full picture of their performance in this area?
+
+### Writing Guidelines
+- **Use active voice** and specific language
+- **Avoid assumptions** about motivations or personality
+- **Focus on behaviors** demonstrated during the interview
+- **Provide context** for any unusual circumstances
+- **Be constructive** in describing development areas
+- **Maintain professionalism** and respect for candidate
+
+## Common Debrief Challenges and Solutions
+
+### Challenge: "I just don't think they'd fit our culture"
+**Solution**: 
+- Ask for specific, observable evidence
+- Define what "culture fit" means in job-relevant terms
+- Challenge assumptions about cultural requirements
+- Focus on ability to collaborate and contribute effectively
+
+### Challenge: Scores vary widely with no clear explanation
+**Solution**:
+- Review if different interviewers assessed different competencies
+- Look for question differences that might explain variance
+- Consider if candidate performance varied across interviews
+- May need additional data gathering or interview
+
+### Challenge: Everyone loved/hated the candidate but can't articulate why
+**Solution**:
+- Push for specific evidence supporting emotional reactions
+- Review competency rubrics together
+- Look for halo/horn effects influencing overall impression
+- Consider unconscious bias training for team
+
+### Challenge: Technical vs. non-technical interviewers disagree
+**Solution**:
+- Clarify which competencies each interviewer was assessing
+- Ensure technical assessments carry appropriate weight
+- Look for different perspectives on same evidence
+- Consider specialist input for technical decisions
+
+### Challenge: Senior interviewer dominates decision making
+**Solution**:
+- Structure discussion to hear from all levels first
+- Ask direct questions to junior interviewers
+- Challenge opinions that lack supporting evidence
+- Remember that assessment ability doesn't correlate with seniority
+
+### Challenge: Team wants to hire but scores don't support it
+**Solution**:
+- Review if rubrics match actual job requirements
+- Check for consistent application of scoring standards
+- Consider if additional competencies need assessment
+- May indicate need for rubric calibration or role requirement review
+
+## Post-Debrief Actions
+
+### Immediate Actions (Same Day)
+- [ ] **Finalize decision documentation** with all evidence
+- [ ] **Communicate decision** to recruiting team
+- [ ] **Schedule candidate feedback** delivery if applicable
+- [ ] **Update interview scheduling** based on decision
+- [ ] **Note any process improvements** needed for future
+
+### Follow-up Actions (Within 1 Week)
+- [ ] **Deliver candidate feedback** (internal or external)
+- [ ] **Update interview feedback** in tracking system
+- [ ] **Schedule any additional interviews** if needed
+- [ ] **Begin offer process** if hiring
+- [ ] **Document lessons learned** for process improvement
+
+### Long-term Actions (Monthly/Quarterly)
+- [ ] **Analyze debrief effectiveness** and decision quality
+- [ ] **Review interviewer calibration** based on decisions
+- [ ] **Update rubrics** based on debrief insights
+- [ ] **Provide additional training** if bias patterns identified
+- [ ] **Share successful practices** with other hiring teams
+
+## Continuous Improvement Framework
+
+### Debrief Effectiveness Metrics
+- **Decision consistency**: Are similar candidates receiving similar decisions?
+- **Time to decision**: Are debriefs completing within planned time?
+- **Participation quality**: Are all interviewers contributing evidence-based input?
+- **Bias incidents**: How often are bias interruptions needed?
+- **Decision satisfaction**: Do participants feel good about the process and outcome?
+
+### Regular Review Process
+- **Monthly**: Review debrief facilitation effectiveness and interviewer feedback
+- **Quarterly**: Analyze decision patterns and potential bias indicators  
+- **Semi-annually**: Update debrief processes based on hiring outcome data
+- **Annually**: Comprehensive review of debrief framework and training needs
+
+### Training and Calibration
+- **New facilitators**: Shadow 3-5 debriefs before leading independently
+- **All facilitators**: Quarterly calibration sessions on bias interruption
+- **Interviewer training**: Include debrief participation expectations
+- **Leadership training**: Ensure hiring managers can facilitate effectively
+
+This guide should be adapted to your organization's specific needs while maintaining focus on evidence-based, unbiased decision making.

package/engineering/skills/interview-system-designer/references/interview-frameworks.md

@@ -0,0 +1,46 @@
+# Interview Frameworks
+
+## Loop Design by Level
+
+### Junior/Mid
+
+- Emphasize fundamentals, debugging, and growth potential.
+- Keep loops concise with coding + behavioral validation.
+
+### Senior
+
+- Add system design and leadership rounds.
+- Evaluate tradeoff quality, mentoring, and cross-team collaboration.
+
+### Staff+
+
+- Focus on architecture direction and organizational impact.
+- Assess strategy, influence, and long-term technical judgment.
+
+## Competency Areas
+
+- Technical depth (implementation, design, quality)
+- Problem solving (ambiguity handling, prioritization)
+- Collaboration (communication, stakeholder alignment)
+- Leadership (ownership, mentoring, influence)
+
+## Scoring Rubric Baseline
+
+- `4`: exceeds level expectations with strong evidence
+- `3`: meets expectations consistently
+- `2`: partial signal with notable gaps
+- `1`: does not meet baseline requirements
+
+## Calibration Guidelines
+
+- Run recurring interviewer calibration sessions.
+- Compare interviewer scoring variance across rounds.
+- Track interview signal against new-hire outcomes.
+- Use structured debriefs with independent scoring before discussion.
+
+## Bias-Reduction Baseline
+
+- Standardize question banks per competency area.
+- Keep scorecards evidence-based and behavior-specific.
+- Use diverse interviewer panels where possible.
+- Require written rationale for strong yes/no recommendations.

package/engineering/skills/interview-system-designer/scripts/interview_planner.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Generate an interview loop plan by role and level."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from typing import Dict, List
+
+BASE_ROUNDS = {
+    "junior": [
+        ("Screen", 45, "Fundamentals and communication"),
+        ("Coding", 60, "Problem solving and code quality"),
+        ("Behavioral", 45, "Collaboration and growth mindset"),
+    ],
+    "mid": [
+        ("Screen", 45, "Fundamentals and ownership"),
+        ("Coding", 60, "Implementation quality"),
+        ("System Design", 60, "Service/component design"),
+        ("Behavioral", 45, "Stakeholder collaboration"),
+    ],
+    "senior": [
+        ("Screen", 45, "Depth and tradeoff reasoning"),
+        ("Coding", 60, "Code quality and testing"),
+        ("System Design", 75, "Scalability and reliability"),
+        ("Leadership", 60, "Mentoring and decision making"),
+        ("Behavioral", 45, "Cross-functional influence"),
+    ],
+    "staff": [
+        ("Screen", 45, "Strategic and technical depth"),
+        ("Architecture", 90, "Org-level design decisions"),
+        ("Technical Strategy", 60, "Long-term tradeoffs"),
+        ("Influence", 60, "Cross-team leadership"),
+        ("Behavioral", 45, "Values and executive communication"),
+    ],
+}
+
+QUESTION_BANK = {
+    "coding": [
+        "Walk through your approach before coding and identify tradeoffs.",
+        "How would you test this implementation for edge cases?",
+        "What would you refactor if this code became a shared library?",
+    ],
+    "system": [
+        "Design this system for 10x traffic growth in 12 months.",
+        "Where are the main failure modes and how would you detect them?",
+        "What components would you scale first and why?",
+    ],
+    "leadership": [
+        "Describe a time you changed technical direction with incomplete information.",
+        "How do you raise the bar for code quality across a team?",
+        "How do you handle disagreement between product and engineering priorities?",
+    ],
+    "behavioral": [
+        "Tell me about a high-stakes mistake and what changed afterward.",
+        "Describe a conflict where you had to influence without authority.",
+        "How do you support underperforming teammates?",
+    ],
+}
+
+
+def normalize_level(level: str) -> str:
+    level = level.strip().lower()
+    if level in {"staff+", "principal", "lead"}:
+        return "staff"
+    if level not in BASE_ROUNDS:
+        raise ValueError(f"Unsupported level: {level}")
+    return level
+
+
+def suggested_questions(round_name: str) -> List[str]:
+    name = round_name.lower()
+    if "coding" in name:
+        return QUESTION_BANK["coding"]
+    if "system" in name or "architecture" in name:
+        return QUESTION_BANK["system"]
+    if "lead" in name or "influence" in name or "strategy" in name:
+        return QUESTION_BANK["leadership"]
+    return QUESTION_BANK["behavioral"]
+
+
+def generate_plan(role: str, level: str) -> Dict[str, object]:
+    normalized = normalize_level(level)
+    rounds = []
+    for idx, (name, minutes, focus) in enumerate(BASE_ROUNDS[normalized], start=1):
+        rounds.append(
+            {
+                "round": idx,
+                "name": name,
+                "duration_minutes": minutes,
+                "focus": focus,
+                "suggested_questions": suggested_questions(name),
+            }
+        )
+    return {
+        "role": role,
+        "level": normalized,
+        "total_rounds": len(rounds),
+        "total_minutes": sum(r["duration_minutes"] for r in rounds),
+        "rounds": rounds,
+    }
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Generate an interview loop plan for a role and level.")
+    parser.add_argument("--role", required=True, help="Role name (e.g., Senior Software Engineer)")
+    parser.add_argument("--level", required=True, help="Level: junior|mid|senior|staff")
+    parser.add_argument("--json", action="store_true", help="Output as JSON")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    plan = generate_plan(args.role, args.level)
+
+    if args.json:
+        print(json.dumps(plan, indent=2))
+    else:
+        print(f"Interview Plan: {plan['role']} ({plan['level']})")
+        print(f"Total rounds: {plan['total_rounds']} | Total time: {plan['total_minutes']} minutes")
+        print("")
+        for r in plan["rounds"]:
+            print(f"Round {r['round']}: {r['name']} ({r['duration_minutes']} min)")
+            print(f"Focus: {r['focus']}")
+            for q in r["suggested_questions"]:
+                print(f"- {q}")
+            print("")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())