@diottodev/runic 1.0.0 → 1.1.0

package/research-ops/skills/market-research/scripts/segmentation_scorer.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""segmentation_scorer.py - Score candidate market segments against Kotler's actionability criteria.
+
+Stdlib-only. Deterministic. NO LLM calls.
+
+Each candidate segment is scored 0-100 across the five Kotler criteria for a useful segment:
+  1. measurable      can you size and identify it?
+  2. substantial     is it large/profitable enough to serve?
+  3. accessible      can you reach it through channels?
+  4. differentiable   does it respond differently from other segments?
+  5. actionable      can you design and execute a program for it?
+
+Segments failing the substantiality or accessibility gates are flagged: a demographic slice
+that is unreachable or too small is not a market segment.
+
+Usage:
+    python3 segmentation_scorer.py --sample
+    python3 segmentation_scorer.py --input segments.json --profile enterprise
+    python3 segmentation_scorer.py --input segments.json --output json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+try:
+    import config_loader as _cfg
+except ImportError:  # pragma: no cover
+    _cfg = None
+
+CRITERIA = ["measurable", "substantial", "accessible", "differentiable", "actionable"]
+WEIGHTS = {"measurable": 0.15, "substantial": 0.25, "accessible": 0.25,
+           "differentiable": 0.20, "actionable": 0.15}
+GATE_FLOOR = 40.0  # substantiality + accessibility gate
+
+# Profiles nudge the substantiality expectation (enterprise tolerates smaller, higher-value segments).
+PROFILES = {
+    "b2b-saas": 1.0,
+    "consumer": 1.0,
+    "enterprise": 0.85,
+    "marketplace": 1.0,
+    "hardware": 1.0,
+    "services": 0.9,
+}
+
+SAMPLE = {
+    "segments": [
+        {"name": "Mid-market HR teams (51-250 emp)",
+         "scores": {"measurable": 85, "substantial": 80, "accessible": 75, "differentiable": 70, "actionable": 80}},
+        {"name": "Solopreneurs who 'might' want analytics",
+         "scores": {"measurable": 40, "substantial": 30, "accessible": 35, "differentiable": 30, "actionable": 40}},
+        {"name": "Enterprise CHROs (1000+ emp)",
+         "scores": {"measurable": 90, "substantial": 95, "accessible": 50, "differentiable": 85, "actionable": 70}},
+    ],
+}
+
+
+def score_segment(seg: dict, sub_mult: float) -> dict:
+    raw = seg.get("scores", {})
+    breakdown = {}
+    composite = 0.0
+    for c in CRITERIA:
+        s = float(raw.get(c, 0.0))
+        if c == "substantial":
+            s = min(100.0, s / sub_mult)  # enterprise: smaller segments still count (divide by <1 raises)
+        breakdown[c] = round(s, 1)
+        composite += s * WEIGHTS[c]
+    flags = []
+    if breakdown["substantial"] < GATE_FLOOR:
+        flags.append("FAILS SUBSTANTIALITY GATE: too small/unprofitable to be a target segment.")
+    if breakdown["accessible"] < GATE_FLOOR:
+        flags.append("FAILS ACCESSIBILITY GATE: no viable channel to reach it.")
+    verdict = "DROP" if flags else ("TARGET" if composite >= 65 else "WATCH")
+    return {
+        "name": seg.get("name", "UNNAMED"),
+        "composite": round(composite, 1),
+        "breakdown": breakdown,
+        "flags": flags,
+        "verdict": verdict,
+    }
+
+
+def evaluate(data: dict, profile: str) -> dict:
+    if profile not in PROFILES:
+        raise ValueError(f"Unknown profile '{profile}'. Choose from {list(PROFILES)}.")
+    mult = PROFILES[profile]
+    scored = sorted((score_segment(s, mult) for s in data.get("segments", [])),
+                    key=lambda x: x["composite"], reverse=True)
+    return {
+        "profile": profile,
+        "segments": scored,
+        "note": "A demographic or firmographic slice is not a segment unless it is substantial AND accessible.",
+    }
+
+
+def _render_human(r: dict) -> str:
+    lines = [f"Segmentation Scoring (profile: {r['profile']})", ""]
+    for s in r["segments"]:
+        lines.append(f"[{s['verdict']}] {s['name']}  — composite {s['composite']}/100")
+        for c, v in s["breakdown"].items():
+            lines.append(f"      {c:16s} {v}")
+        for f in s["flags"]:
+            lines.append(f"      ! {f}")
+        lines.append("")
+    lines.append(f"note: {r['note']}")
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description="Score market segments against Kotler's 5 criteria.")
+    p.add_argument("--input", help="Path to JSON with segments[]")
+    p.add_argument("--profile", default=None, choices=list(PROFILES),
+                   help="overrides onboarding default_profile")
+    p.add_argument("--output", choices=["human", "json"], default="human")
+    p.add_argument("--sample", action="store_true", help="use the embedded sample")
+    args = p.parse_args(argv)
+
+    conf = _cfg.load_config() if _cfg else {}
+    profile = args.profile or conf.get("default_profile", "b2b-saas")
+    data = SAMPLE if (args.sample or not args.input) else json.load(open(args.input))
+    try:
+        result = evaluate(data, profile)
+    except ValueError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+
+    if args.output == "json":
+        print(json.dumps(result, indent=2))
+    else:
+        print(_render_human(result))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/research-ops/skills/product-research/SKILL.md

@@ -0,0 +1,145 @@
+---
+name: product-research
+description: Use when planning and synthesizing product/user research as a method-and-repository discipline — selecting the right method for the goal (generative interviews vs usability test vs concept test vs validation), computing method-based saturation/sample size with an explicit confidence level, or synthesizing coded observations into insights while flagging single-source anecdotes. Never fabricates user insight; an insight requires recurrence across independent participants. Distinct from product-team/ux-researcher-designer (persona/journey artifacts), product-discovery (discovery-sprint planning), and experiment-designer (live A/B) — this is the research-ops method + insight-repository layer.
+version: 2.9.0
+author: claude-code-skills
+license: MIT
+tags: [research-ops, product-research, ux-research, jtbd, usability, saturation, insight-synthesis, research-repository]
+compatible_tools: [claude-code, codex-cli, cursor, antigravity, opencode, gemini-cli]
+---
+
+# product-research
+
+Product / user research as an operational discipline: choosing the right method, sizing it honestly, and synthesizing findings into governed insights. The core rule: **method must match the goal**, and **an insight requires recurrence across independent participants** — a single quote is an anecdote.
+
+## Purpose
+
+Product researchers, ResearchOps teams, and PMs running discovery need method rigor and an insight repository they can trust. This skill structures three decisions:
+
+Three deterministic tools:
+
+1. `study_designer.py` — Maps (research goal × product stage) to an appropriate method and emits a method-matched plan skeleton (objective, participant criteria, guide structure, success criteria). Redirects live A/B to `product-team/experiment-designer`.
+2. `saturation_planner.py` — Method-based sample guidance with an explicit **confidence label**: Nielsen problem-discovery (5/segment), Guest et al. thematic saturation (~12), and evaluative coverage. Never claims a prevalence rate from a small-n usability test.
+3. `insight_synthesizer.py` — Clusters coded observations by tag, counts distinct participants, ranks by cross-participant recurrence, and flags any candidate below the source threshold as an **ANECDOTE**, never promoting it to an insight.
+
+## When to use
+
+Invoke this skill when:
+
+- You are planning a study and need the method to match the goal (generative vs evaluative vs validation).
+- You need a defensible sample size / saturation rationale with a stated confidence.
+- You have raw coded observations and need to synthesize insights without over-claiming.
+- You are setting up or auditing a research repository and need the insight-vs-observation discipline.
+
+**Do NOT use this skill to**: generate personas / journey maps (use `product-team/ux-researcher-designer`), plan a discovery sprint or validate an opportunity (use `product-team/product-discovery`), design or analyze a live product A/B experiment (use `product-team/experiment-designer`), or do market sizing / surveys (use the `market-research` sibling).
+
+## Workflow
+
+1. **Frame the study** — Fill `assets/research_plan_template.md` (research questions, method rationale, participant criteria, analysis plan, repository tagging scheme).
+2. **Pick the method** — Run `study_designer.py --goal {discovery|evaluative|validation} --stage {concept|prototype|beta|live} --profile {b2b-saas|consumer-app|enterprise|marketplace|hardware|platform}`. Honor the redirect if it routes to experiment-designer.
+3. **Size it** — Run `saturation_planner.py --method {usability|thematic|evaluative-coverage} --segments N`. Record the confidence label and limits.
+4. **Synthesize** — After fielding, code observations and run `insight_synthesizer.py --input observations.json --min-sources 3`. Treat ANECDOTE-flagged clusters as signals to probe, not findings to ship.
+5. **File in the repository** — Tag insights to the atomic schema at synthesis time, with their evidence and confidence.
+
+## Scripts
+
+| Script | Purpose | Profiles |
+|---|---|---|
+| `scripts/study_designer.py` | (goal × stage) → method + plan skeleton | b2b-saas, consumer-app, enterprise, marketplace, hardware, platform |
+| `scripts/saturation_planner.py` | Method-based sample guidance + confidence | n/a (method-driven) |
+| `scripts/insight_synthesizer.py` | Cluster observations, flag anecdotes | n/a (evidence-driven) |
+
+All three: stdlib-only, `--help`, `--sample`, `--output {human,json}`.
+
+## Onboarding & customization
+
+Run the onboarding questionnaire **once before you start** — it captures your defaults so every tool in this skill is pre-configured. Customization is the point: the answers actually change tool behavior (e.g. the insight source-threshold).
+
+```bash
+python3 scripts/onboard.py            # interactive (also: --defaults, --set key=value, --reset)
+python3 scripts/onboard.py --show     # see the questions + current effective config
+```
+
+Answers are saved to `~/.config/research-ops/product-research.json` (global) or `./.research-ops/product-research.json` (`--scope project`) and are read automatically by `config_loader.py`. They set the default product **profile**, the **insight source-threshold** (how many independent participants make a finding an insight, not an anecdote), the default **saturation method**, and the **high-stakes** flag. CLI flags always override saved config; `RESEARCH_OPS_NO_CONFIG=1` ignores it.
+
+**The four questions:** product profile · insight source-threshold · saturation method · high-stakes flag.
+
+## Optimize with autoresearch (opt-in)
+
+This skill ships an **isolated, opt-in** bridge to `engineering/autoresearch-agent`. Only when you ask to "optimize the synthesis" / "run a loop" does an autoresearch experiment iteratively refine the coding/clustering of a fixed evidence set so more cross-participant patterns surface. `scripts/ar_evaluator.py` is the ground-truth evaluator; it prints `validated_insights: <int>` (higher is better). It optimizes the **coding**, never fabricates evidence.
+
+```bash
+/ar:setup --domain custom --name insight-synthesis \
+  --target observations.json \
+  --eval "python3 ar_evaluator.py --target observations.json" \
+  --metric validated_insights --direction higher
+/ar:loop custom/insight-synthesis
+```
+
+Isolated: no hard dependency — autoresearch runs only on demand, and the loop edits `observations.json`, never the evaluator.
+
+## References
+
+- `references/research_methods_canon.md` — Portigal *Interviewing Users*; Christensen/Ulwick JTBD; Rohrer's UX-research methods landscape (NN/g); Sauro & Lewis *Quantifying the User Experience*; Goodman/Kuniavsky.
+- `references/sampling_and_saturation.md` — Nielsen "test with 5 users"; Guest, Bunce & Johnson saturation; Faulkner on more-than-5; Sauro usability sample size; Braun & Clarke thematic analysis.
+- `references/repository_and_synthesis.md` — ResearchOps / atomic research (Tomer Sharon "Polaris"); insight-vs-observation discipline; repository governance; affinity mapping; democratization guardrails.
+
+## Assumptions
+
+- Method selection assumes you can name the goal honestly; if the goal is fuzzy, grill it first (the goal drives everything).
+- Saturation guidance is method-based, not a power calculation — usability tests find problems, not prevalence rates.
+- The synthesizer counts evidence you provide; coding quality is upstream of it. Garbage tags → garbage clusters.
+- The insight threshold (`--min-sources`) defaults to 3; raise it for high-stakes or heterogeneous populations.
+
+## Anti-patterns
+
+- **Mismatching method to goal.** A usability test cannot discover unmet needs; an interview cannot measure task success.
+- **Reporting usability problems as percentages.** Small-n tests surface problems, not population rates.
+- **Promoting an anecdote to an insight.** One participant is a signal to probe, not a finding.
+- **Framing interview questions as feature reactions.** Probe the job-to-be-done and recent real behavior, not hypothetical opinions.
+- **Synthesizing without a repository scheme.** Tag at synthesis time, or insights rot unfindable.
+
+## Distinct from
+
+| Neighbor | Scope | Difference |
+|---|---|---|
+| `product-team/ux-researcher-designer` | Personas, journey maps, usability frameworks tied to design output | That produces **artifacts**; this is **method + repository discipline** |
+| `product-team/product-discovery` | Opportunity validation, discovery-sprint planning | That plans **discovery sprints**; this designs and synthesizes the **research** |
+| `product-team/experiment-designer` | Live product A/B hypothesis + sample size | That runs **live experiments**; this runs **qualitative/evaluative research** |
+| `market-research` (sibling) | Market sizing, surveys, segmentation | That studies **the market**; this studies **users** |
+
+## Quick examples
+
+```bash
+python3 scripts/study_designer.py --sample
+python3 scripts/saturation_planner.py --method thematic --segments 3
+python3 scripts/insight_synthesizer.py --sample --min-sources 3
+```
+
+The synthesizer sample correctly promotes "import-confusion" (3 independent participants) to INSIGHT and flags "wants-slack" (1 participant) as an ANECDOTE.
+
+## Forcing-question library (Matt Pocock grill discipline)
+
+Walked one at a time by `/cs:grill-research-ops` or the orchestrator. Recommended answer + canon citation per question. Never bundled.
+
+1. **"Is this study generative (discover problems) or evaluative (test a solution)?"**
+   Recommended: name it first — the method follows from the goal.
+   Canon: Rohrer, *When to Use Which User-Experience Research Methods* (NN/g).
+
+2. **"What's your sample size and saturation rationale — and at what confidence?"**
+   Recommended: method-based n (5/segment usability; ~12 for thematic saturation), state the confidence.
+   Canon: Nielsen; Guest, Bunce & Johnson (2006); Faulkner (2003).
+
+3. **"How many independent participants support each insight — or is it a single-source anecdote?"**
+   Recommended: require recurrence across ≥3 sources before calling it an insight; flag singletons.
+   Canon: atomic research / ResearchOps; Braun & Clarke thematic analysis.
+
+4. **"Are your interview / usability tasks framed as outcomes (jobs) or as feature reactions?"**
+   Recommended: frame around the job-to-be-done and recent real behavior, not hypothetical opinion.
+   Canon: Christensen/Ulwick Jobs-to-be-Done; Portigal *Interviewing Users*.
+
+5. **"Where does this land in the repository, and how is it tagged for reuse?"**
+   Recommended: tag to the atomic schema at synthesis time, not later.
+   Canon: Tomer Sharon, *Polaris* / ResearchOps repository practice.
+
+Walk depth-first. Lock 1-2 before opening 3-5. After all are answered, invoke `study_designer.py` → `saturation_planner.py` → (after fielding) `insight_synthesizer.py`.

package/research-ops/skills/product-research/assets/research_plan_template.md

@@ -0,0 +1,50 @@
+# Product Research Plan — Template
+
+> Fill this before running the tools. Method must match the goal. An insight requires
+> recurrence across independent participants — a single quote is an anecdote.
+
+## 1. Study identification
+- Study name:
+- Product / feature:
+- Stage: [concept | prototype | beta | live]
+- Profile: [b2b-saas | consumer-app | enterprise | marketplace | hardware | platform]
+
+## 2. Goal & questions
+- Goal: [discovery (generative) | evaluative | validation]
+- Research questions (3-5, answerable, not leading):
+- The product decision this informs:
+
+## 3. Method (from `study_designer.py`)
+- Recommended method:
+- Why it matches the goal:
+- (If live A/B → route to product-team/experiment-designer.)
+
+## 4. Participants
+- Target segment(s) + screener (screen for the job, not a job title):
+- Per-segment recruiting if reporting per segment? [yes/no]
+- Exclusions (internal, biased, repeat):
+
+## 5. Sample & saturation (from `saturation_planner.py`)
+- Method: [usability | thematic | evaluative-coverage]
+- n per segment + total:
+- Confidence label + limits:
+
+## 6. Study guide skeleton
+1.
+2.
+3.
+4.
+5.
+
+## 7. Analysis & synthesis
+- Coding / tagging scheme (atomic taxonomy):
+- Insight threshold (min distinct participants): ___ (default 3)
+- Synthesis tool: `insight_synthesizer.py`
+
+## 8. Repository
+- Where insights are filed + tagging taxonomy:
+- Evidence linked to each insight? [yes — required]
+- Confidence field per insight? [yes — required]
+
+## 9. Confidence statement
+- What this study can and cannot support:

package/research-ops/skills/product-research/references/repository_and_synthesis.md

@@ -0,0 +1,28 @@
+# Research Repository and Synthesis
+
+Reference for turning observations into governed insights. Pairs with `insight_synthesizer.py`.
+
+## Observation vs insight
+
+The foundational discipline of ResearchOps is the distinction between an **observation** (a single piece of evidence — one participant did or said one thing) and an **insight** (a pattern that recurs across independent sources and carries an implication). Promoting an observation to an insight because it was vivid or confirmed a prior is the cardinal sin of synthesis. The synthesizer enforces a source threshold: a candidate supported by fewer than the threshold of distinct participants is labeled an ANECDOTE and is never promoted.
+
+## Atomic research
+
+Tomer Sharon's **atomic research** model (and the "Polaris" repository concept) decomposes research into reusable units: *Experiments → Facts (observations) → Insights → Recommendations*. Facts are tagged and stored so that insights can be traced back to evidence and reused across studies. The payoff is a repository where a claim can always be drilled down to the observations that support it — and where the same evidence can support future questions.
+
+## Affinity mapping
+
+The classic synthesis technique is affinity mapping: cluster observations into emergent themes bottom-up, then name the themes. The `insight_synthesizer.py` tool is a deterministic, tag-based proxy for this — it clusters by the codes you assign and ranks by cross-participant recurrence. The human still does the interpretive naming; the tool enforces the counting discipline.
+
+## Repository governance and democratization
+
+As organizations democratize research (PMs and designers running their own studies), the repository becomes the guardrail. Governance practices: a consistent tagging taxonomy, evidence linked to every insight, a confidence field, and a review step before an insight is marked "validated." Without governance, democratized research produces a pile of unsearchable anecdotes; with it, the repository compounds in value.
+
+## Sources
+
+1. Sharon, T., *Validating Product Ideas Through Lean User Research* (Rosenfeld, 2016) and the atomic-research / Polaris model.
+2. ResearchOps Community, *Research Repositories* and *Democratization* working-group reports.
+3. Braun, V., & Clarke, V., *Thematic Analysis: A Practical Guide* (Sage, 2022).
+4. Beyer, H., & Holtzblatt, K., *Contextual Design* (1998) — affinity diagramming.
+5. Dovetail / EnjoyHQ practitioner guides on insight repositories and tagging taxonomies.
+6. Kaplan, K., *Taxonomy 101* and *Research Repositories* — Nielsen Norman Group.

package/research-ops/skills/product-research/references/research_methods_canon.md

@@ -0,0 +1,34 @@
+# Product Research Methods Canon
+
+Reference for method selection. Pairs with `study_designer.py`.
+
+## The two-axis map
+
+UX/product research methods sort along two axes (Rohrer, NN/g): **attitudinal vs behavioral** (what people say vs what they do) and **qualitative vs quantitative** (why/how vs how-many). The single most important pre-method decision is the **goal**:
+
+- **Generative (discovery)** — you don't yet know the problem. Methods: semi-structured interviews, contextual inquiry, diary studies. Output: themes, unmet needs, jobs-to-be-done.
+- **Evaluative** — you have a solution and want to know if it works. Methods: moderated/unmoderated usability tests, concept tests. Output: task-success, severity-rated problems.
+- **Validation** — you want to confirm demand/desirability before building. Methods: surveys, preference tests, fake-door tests, and (when live) A/B experiments.
+
+Picking an evaluative method for a generative goal — "let's usability-test our way to product strategy" — is the most common and most expensive error.
+
+## Interviewing discipline
+
+Steve Portigal's *Interviewing Users* is the operative craft reference: ask about **recent, specific, real behavior** ("tell me about the last time you…"), not hypotheticals or opinions ("would you use…"). People are unreliable narrators of their future selves but good storytellers of their past.
+
+## Jobs-to-be-Done
+
+Christensen's and Ulwick's JTBD reframes research around the **progress a person is trying to make** in a circumstance, not their demographics or feature preferences. Outcome-Driven Innovation (Ulwick) operationalizes this into measurable desired outcomes — a bridge between qualitative discovery and quantitative validation.
+
+## Mixed methods
+
+Strong research triangulates: qualitative discovery surfaces hypotheses; quantitative validation sizes them. Sauro & Lewis (*Quantifying the User Experience*) provides the statistical backbone for turning usability observations into defensible metrics (task time, completion, SUS) without over-claiming from small samples.
+
+## Sources
+
+1. Portigal, S., *Interviewing Users*, 2nd ed. (Rosenfeld, 2023).
+2. Christensen, Hall, Dillon & Duncan, *Competing Against Luck* (2016) — Jobs-to-be-Done.
+3. Ulwick, A., *Jobs to Be Done: Theory to Practice* (2016) — Outcome-Driven Innovation.
+4. Rohrer, C., *When to Use Which User-Experience Research Methods* — Nielsen Norman Group.
+5. Sauro, J., & Lewis, J., *Quantifying the User Experience*, 2nd ed. (Morgan Kaufmann, 2016).
+6. Goodman, Kuniavsky & Moed, *Observing the User Experience*, 2nd ed. (2012).

package/research-ops/skills/product-research/references/sampling_and_saturation.md

@@ -0,0 +1,29 @@
+# Sampling and Saturation
+
+Reference for how many participants. Pairs with `saturation_planner.py`.
+
+## Usability: the "5 users" result
+
+Nielsen and Landauer's model says the proportion of usability problems found with n users is 1 − (1 − p)ⁿ, where p is the average probability that a single user surfaces a given problem (~0.31 in their data). At n = 5, that is ~85% of problems — hence "test with 5 users." Two crucial caveats the planner enforces:
+
+1. **Per segment.** The 5-user result holds *within a homogeneous user group*. If you have distinct segments that behave differently, you need ~5 per segment.
+2. **Problems, not rates.** A small-n usability test finds *whether* a problem exists; it cannot estimate the *prevalence* of that problem in the population. Never report "60% of users struggled" from a 5-person test.
+
+Faulkner (2003) showed real variance: while the average across many 5-person samples is ~85%, individual 5-person runs ranged from ~55% to 100%. When stakes or heterogeneity are high, run more.
+
+## Qualitative: thematic saturation
+
+For interview-based thematic research, Guest, Bunce & Johnson (2006) found that **saturation** — the point where new interviews stop yielding new themes — typically occurs by ~12 interviews in a homogeneous group, with the basic elements present by ~6. Saturation is **observed, not guaranteed**: track the new-theme rate and stop when it flattens, rather than committing to a fixed n blindly. Heterogeneous populations need more, and per-group saturation applies just as in usability.
+
+## Reporting confidence honestly
+
+The planner attaches a confidence label (LOW / MODERATE / MODERATE-HIGH) and explicit limits to every plan, because the failure mode in product research is not too-small samples per se — it is **over-claiming** from whatever sample you ran. State the method, the n, and what the method can and cannot support.
+
+## Sources
+
+1. Nielsen, J., & Landauer, T., *A mathematical model of the finding of usability problems* — INTERCHI 1993.
+2. Nielsen, J., *Why You Only Need to Test with 5 Users* — NN/g (2000).
+3. Faulkner, L., *Beyond the five-user assumption* — Behavior Research Methods 2003;35:379-383.
+4. Guest, G., Bunce, A., & Johnson, L., *How many interviews are enough?* — Field Methods 2006;18:59-82.
+5. Braun, V., & Clarke, V., *Using thematic analysis in psychology* — Qual Res Psychol 2006;3:77-101.
+6. Sauro, J., & Lewis, J., *Quantifying the User Experience*, 2nd ed. (2016) — confidence intervals for small samples.

package/research-ops/skills/product-research/scripts/ar_evaluator.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""ar_evaluator.py - Autoresearch evaluator for the product-research skill (OPT-IN).
+
+Stdlib-only. The ISOLATED bridge to engineering/autoresearch-agent. It does NOT call
+autoresearch; it is the ground-truth evaluator an autoresearch loop runs after editing
+the target coded-observations file. It reads an observations JSON, runs insight_synthesizer
+at the configured source threshold, and prints ONE metric line:
+
+    validated_insights: <int>   (higher is better — clusters that clear the source threshold)
+
+This optimizes the CODING/synthesis of a fixed evidence set (merging/splitting tags so
+cross-participant patterns surface) — not the evidence itself. The user opts in explicitly:
+    /ar:setup --domain custom --name insight-synthesis \\
+      --target observations.json --eval "python3 ar_evaluator.py --target observations.json" \\
+      --metric validated_insights --direction higher
+
+Direct use:
+    python3 ar_evaluator.py --sample
+    python3 ar_evaluator.py --target observations.json --min-sources 3
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import config_loader as cfg  # noqa: E402
+import insight_synthesizer as isyn  # noqa: E402
+
+METRIC = "validated_insights"
+
+
+def main(argv: list[str] | None = None) -> int:
+    c = cfg.load_config()
+    p = argparse.ArgumentParser(description="Autoresearch evaluator: count of validated insights.")
+    p.add_argument("--target", help="path to observations JSON (or env AR_TARGET)")
+    p.add_argument("--min-sources", type=int, default=None, help="overrides onboarding insight_min_sources")
+    p.add_argument("--sample", action="store_true")
+    args = p.parse_args(argv)
+
+    min_sources = args.min_sources if args.min_sources is not None else int(c.get("insight_min_sources", 3))
+
+    if args.sample:
+        data = isyn.SAMPLE
+    else:
+        target = args.target or os.environ.get("AR_TARGET")
+        if not target:
+            print("error: provide --target <observations.json> or set AR_TARGET", file=sys.stderr)
+            return 2
+        try:
+            with open(target) as f:
+                data = json.load(f)
+        except (OSError, json.JSONDecodeError) as e:
+            print(f"{METRIC}: N/A")
+            print(f"error: {e}", file=sys.stderr)
+            return 1
+
+    result = isyn.synthesize(data, min_sources)
+    count = sum(1 for c2 in result["candidates"] if c2["classification"] == "INSIGHT")
+    print(f"{METRIC}: {count}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/research-ops/skills/product-research/scripts/config_loader.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""config_loader.py - Customization loader for the product-research skill.
+
+Stdlib-only. Importable from the skill's other scripts. Precedence (highest wins):
+  1. Project config:  <cwd>/.research-ops/product-research.json
+  2. Global config:   ~/.config/research-ops/product-research.json
+  3. Built-in DEFAULTS
+
+Onboarding answers (written by onboard.py) live in these files; every tool in this
+skill reads them so the user's customization applies automatically.
+Set RESEARCH_OPS_NO_CONFIG=1 to ignore saved config.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+SKILL = "product-research"
+
+GLOBAL_CONFIG_DIR = Path.home() / ".config" / "research-ops"
+GLOBAL_CONFIG_PATH = GLOBAL_CONFIG_DIR / f"{SKILL}.json"
+PROJECT_CONFIG_DIRNAME = ".research-ops"
+
+DEFAULTS: dict[str, Any] = {
+    "version": 1,
+    "skill": SKILL,
+    "default_profile": "b2b-saas",
+    "insight_min_sources": 3,
+    "default_method": "usability",
+    "stakes_high": False,
+    "setup_completed_at": None,
+}
+
+
+def project_config_path(cwd: Path | None = None) -> Path:
+    cwd = cwd or Path.cwd()
+    return cwd / PROJECT_CONFIG_DIRNAME / f"{SKILL}.json"
+
+
+def _read_json(path: Path) -> dict[str, Any] | None:
+    try:
+        with path.open(encoding="utf-8") as f:
+            data = json.load(f)
+        return data if isinstance(data, dict) else None
+    except (FileNotFoundError, json.JSONDecodeError, OSError):
+        return None
+
+
+def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+    out = dict(base)
+    for k, v in override.items():
+        if isinstance(v, dict) and isinstance(out.get(k), dict):
+            out[k] = _deep_merge(out[k], v)
+        else:
+            out[k] = v
+    return out
+
+
+def load_config(cwd: Path | None = None) -> dict[str, Any]:
+    config = dict(DEFAULTS)
+    if os.environ.get("RESEARCH_OPS_NO_CONFIG") == "1":
+        return config
+    global_cfg = _read_json(GLOBAL_CONFIG_PATH)
+    if global_cfg:
+        config = _deep_merge(config, global_cfg)
+    project_cfg = _read_json(project_config_path(cwd))
+    if project_cfg:
+        config = _deep_merge(config, project_cfg)
+    return config
+
+
+def setup_completed() -> bool:
+    cfg = _read_json(GLOBAL_CONFIG_PATH) or _read_json(project_config_path())
+    return bool(cfg and cfg.get("setup_completed_at"))
+
+
+def write_config(config: dict[str, Any], scope: str = "global", cwd: Path | None = None) -> Path:
+    path = project_config_path(cwd) if scope == "project" else GLOBAL_CONFIG_PATH
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        json.dump(config, f, indent=2, sort_keys=True)
+    return path
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description=f"Inspect {SKILL} customization config.")
+    p.add_argument("--show", action="store_true", help="Print the effective config")
+    p.add_argument("--status", action="store_true", help="Print setup status + paths")
+    p.add_argument("--sample", action="store_true", help="Print the built-in defaults")
+    args = p.parse_args(argv)
+
+    if args.sample:
+        print(json.dumps(DEFAULTS, indent=2, sort_keys=True))
+    elif args.status:
+        print(json.dumps({
+            "skill": SKILL,
+            "global_config_path": str(GLOBAL_CONFIG_PATH),
+            "global_config_exists": GLOBAL_CONFIG_PATH.exists(),
+            "project_config_path": str(project_config_path()),
+            "project_config_exists": project_config_path().exists(),
+            "setup_completed": setup_completed(),
+        }, indent=2))
+    else:
+        print(json.dumps(load_config(), indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())