luckiest-co 1.0.59 → 1.0.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/skills/churn-prevention/CHANGELOG.md +3 -0
- package/skills/churn-prevention/SKILL.md +5 -3
- package/skills/churn-prevention/scripts/test_triage.py +100 -0
- package/skills/churn-prevention/scripts/triage.py +151 -0
- package/skills/community-marketing/CHANGELOG.md +3 -0
- package/skills/community-marketing/SKILL.md +5 -4
- package/skills/community-marketing/scripts/laya_tag.py +180 -0
- package/skills/community-marketing/scripts/test_laya_tag.py +113 -0
- package/skills/customer-research/CHANGELOG.md +3 -0
- package/skills/customer-research/SKILL.md +17 -3
- package/skills/customer-research/scripts/test_triage.py +100 -0
- package/skills/customer-research/scripts/triage.py +151 -0
- package/skills/model-router/CHANGELOG.md +10 -0
- package/skills/model-router/SKILL.md +12 -3
- package/skills/model-router/scripts/route.py +120 -20
- package/skills/model-router/scripts/test_route.py +88 -0
- package/skills/plan/SKILL.md +1 -1
- package/skills/research/CHANGELOG.md +3 -0
- package/skills/research/SKILL.md +5 -3
- package/skills/research/scripts/laya_tag.py +180 -0
- package/skills/research/scripts/test_laya_tag.py +113 -0
- package/skills/skill-audit-security/CHANGELOG.md +3 -0
- package/skills/skill-audit-security/SKILL.md +15 -3
- package/skills/skill-audit-security/scripts/laya_tag.py +180 -0
- package/skills/skill-audit-security/scripts/test_laya_tag.py +113 -0
- package/skills/social/CHANGELOG.md +3 -0
- package/skills/social/SKILL.md +15 -3
- package/skills/social/scripts/laya_tag.py +180 -0
- package/skills/social/scripts/test_laya_tag.py +113 -0
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# Changelog — luckiest-customer-research
|
|
2
2
|
|
|
3
|
+
## 1.3.0 — 2026-11-01
|
|
4
|
+
- Optional Laya tagging: `scripts/triage.py` tags reviews, tickets and survey answers locally (intent, refund, frustration, urgency, churn risk) when the user prefers Laya (`{"laya": true}` or `LUCKIEST_LAYA=1`). Exit 2 means skip it; the skill works as before without Laya.
|
|
5
|
+
|
|
3
6
|
## 1.2.0 — 2026-10-08
|
|
4
7
|
Upstream sync with coreyhaines31/marketingskills dda3841 (issue #112).
|
|
5
8
|
|
|
@@ -3,7 +3,7 @@ name: customer-research
|
|
|
3
3
|
description: "When the user wants to conduct, analyze, or synthesize customer research. Use when the user mentions \"customer research,\" \"ICP research,\" \"talk to customers,\" \"analyze transcripts,\" \"customer interviews,\" \"survey analysis,\" \"support ticket analysis,\" \"voice of customer,\" \"VOC,\" \"build personas,\" \"customer personas,\" \"jobs to be done,\" \"JTBD,\" \"what do customers say,\" \"what are customers struggling with,\" \"Reddit mining,\" \"G2 reviews,\" \"review mining,\" \"digital watering holes,\" \"community research,\" \"forum research,\" \"competitor reviews,\" \"customer sentiment,\" or \"find out why customers churn/convert/buy.\" Use for both analyzing existing research assets AND gathering new research from online sources. For writing copy informed by research, see luckiest-copywriting. For acting on research to improve pages, see luckiest-cro."
|
|
4
4
|
license: See ATTRIBUTION.md
|
|
5
5
|
metadata:
|
|
6
|
-
version: "1.
|
|
6
|
+
version: "1.3.0"
|
|
7
7
|
listing_id: luckiest-customer-research
|
|
8
8
|
author: luckiest
|
|
9
9
|
---
|
|
@@ -16,9 +16,9 @@ This skill acts as an expert customer researcher. Its goal is to help uncover wh
|
|
|
16
16
|
|
|
17
17
|
## Staying current
|
|
18
18
|
|
|
19
|
-
On activation, call the Luckiest MCP check_updates tool with { listingId: "8e289fdd-4950-4503-ae10-c1426e861a12", installedSemver: "1.
|
|
19
|
+
On activation, call the Luckiest MCP check_updates tool with { listingId: "8e289fdd-4950-4503-ae10-c1426e861a12", installedSemver: "1.3.0" }. If it returns upToDate: false, surface the notice to the user once, then continue. Do nothing further if upToDate: true. Never block on this check — if the tool is unavailable, proceed.
|
|
20
20
|
|
|
21
|
-
When the skill's work is done, call the Luckiest MCP report_usage tool once with { listing_id: "luckiest-customer-research", skill_version: "1.
|
|
21
|
+
When the skill's work is done, call the Luckiest MCP report_usage tool once with { listing_id: "luckiest-customer-research", skill_version: "1.3.0", matched: true, success: <true if the skill completed, false otherwise> }. Metadata only, never prompt text. Never block on it — if the tool is unavailable, skip silently.
|
|
22
22
|
|
|
23
23
|
## Before Starting
|
|
24
24
|
|
|
@@ -66,6 +66,20 @@ Most engagements combine both. Establish which mode applies before proceeding.
|
|
|
66
66
|
- Passives and detractors are higher signal than promoters for improvement work
|
|
67
67
|
- Pair scores with verbatims — a 9 with a specific complaint beats a 10 with no comment
|
|
68
68
|
|
|
69
|
+
### Optional: fast tagging with Laya
|
|
70
|
+
|
|
71
|
+
When there are more than about 30 short messages (reviews, tickets, survey answers, call notes split one per line), try tagging them locally first:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
python3 scripts/triage.py messages.txt # one message per line
|
|
75
|
+
python3 scripts/triage.py reviews.csv --column body
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
- It runs only when the user prefers Laya (`{"laya": true}` in `~/.luckiest/config.json`, or `LUCKIEST_LAYA=1`) and Laya is installed.
|
|
79
|
+
- Exit code 2 means Laya is off, missing, or failed. Skip this step and work through the messages by hand as below. Nothing else changes.
|
|
80
|
+
- On success it prints intent counts, the share of messages that are urgent, ask for a refund, or hint at leaving, average frustration (0 to 3), and up to 20 flagged messages. Use the counts for frequency and intensity scoring, and read the flagged messages first for pains and money quotes. Laya tags support intent, not jobs to be done, so still run the extraction framework on what you read.
|
|
81
|
+
- Messages Laya is unsure about are counted as `unsure`. Read a sample of them by hand.
|
|
82
|
+
|
|
69
83
|
### Extraction Framework
|
|
70
84
|
|
|
71
85
|
For each asset, extract:
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Run: python3 -m unittest scripts/test_triage.py (from the skill directory)."""
|
|
2
|
+
import io
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import subprocess
|
|
6
|
+
import sys
|
|
7
|
+
import tempfile
|
|
8
|
+
import unittest
|
|
9
|
+
from unittest import mock
|
|
10
|
+
|
|
11
|
+
sys.path.insert(0, os.path.dirname(__file__))
|
|
12
|
+
import triage # noqa: E402
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def ans(intent="refund", conf=0.9, frustration=2.5, urgent=0.1, refund=0.9, churn=0.1):
|
|
16
|
+
return {"intent": {"choice": intent, "answer_confidence": conf},
|
|
17
|
+
"frustration": {"score": frustration}, "is_urgent": {"noul": urgent},
|
|
18
|
+
"refund_requested": {"noul": refund}, "churn_risk": {"noul": churn}}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def done(rows, code=0):
|
|
22
|
+
return subprocess.CompletedProcess([], code, "\n".join(json.dumps({"answers": a}) for a in rows), "")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Tagging(unittest.TestCase):
|
|
26
|
+
def test_tag_and_unsure(self):
|
|
27
|
+
self.assertEqual(triage.tag(ans()), {"intent": "refund", "frustration": 2.5,
|
|
28
|
+
"urgent": False, "refund": True, "churn_risk": False})
|
|
29
|
+
self.assertIsNone(triage.tag(ans(conf=0.2)))
|
|
30
|
+
self.assertIsNone(triage.tag({"intent": {}}))
|
|
31
|
+
|
|
32
|
+
def test_summary(self):
|
|
33
|
+
msgs = ["give my money back now", "how do I export?", "switching to a rival", "???"]
|
|
34
|
+
rows = [ans(), ans("information", frustration=0.2, refund=0.1),
|
|
35
|
+
ans("cancellation", frustration=2.0, refund=0.1, churn=0.9), ans(conf=0.1)]
|
|
36
|
+
s = triage.summarize(msgs, rows)
|
|
37
|
+
self.assertEqual((s["messages"], s["tagged"], s["unsure"]), (4, 3, 1))
|
|
38
|
+
self.assertEqual(s["intent"]["refund"], 1)
|
|
39
|
+
self.assertEqual((s["refund_pct"], s["churn_risk_pct"]), (33, 33))
|
|
40
|
+
self.assertEqual([f["row"] for f in s["flagged"]], [3, 1]) # leaving + angry outranks refund
|
|
41
|
+
self.assertEqual(s["flagged"][0]["text"], "switching to a rival")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Fallback(unittest.TestCase):
|
|
45
|
+
def call(self, run=None, side_effect=None, exe="/bin/laya", env="1"):
|
|
46
|
+
with mock.patch.dict(os.environ, {"LUCKIEST_LAYA": env}), \
|
|
47
|
+
mock.patch.object(triage, "laya_bin", return_value=exe), \
|
|
48
|
+
mock.patch.object(triage.subprocess, "run", return_value=run, side_effect=side_effect) as m:
|
|
49
|
+
try:
|
|
50
|
+
return triage.laya_answers(["a", "b"]), m
|
|
51
|
+
except triage.Unavailable as e:
|
|
52
|
+
return e, m
|
|
53
|
+
|
|
54
|
+
def test_one_call_when_on(self):
|
|
55
|
+
out, m = self.call(done([ans(), ans()]))
|
|
56
|
+
self.assertEqual(len(out), 2)
|
|
57
|
+
self.assertEqual(m.call_count, 1)
|
|
58
|
+
|
|
59
|
+
def test_unavailable_cases(self):
|
|
60
|
+
cases = [
|
|
61
|
+
dict(env="0"), # not preferred: Laya never runs
|
|
62
|
+
dict(exe=None), # not installed
|
|
63
|
+
dict(side_effect=subprocess.TimeoutExpired("laya", 60)),
|
|
64
|
+
dict(run=done([], code=1)),
|
|
65
|
+
dict(run=subprocess.CompletedProcess([], 0, "not json", "")),
|
|
66
|
+
dict(run=done([ans()])), # wrong row count
|
|
67
|
+
]
|
|
68
|
+
for kw in cases:
|
|
69
|
+
out, m = self.call(**kw)
|
|
70
|
+
self.assertIsInstance(out, triage.Unavailable, kw)
|
|
71
|
+
out, m = self.call(env="0")
|
|
72
|
+
m.assert_not_called()
|
|
73
|
+
|
|
74
|
+
def test_off_by_default(self):
|
|
75
|
+
with mock.patch.dict(os.environ, {"LUCKIEST_LAYA": ""}), \
|
|
76
|
+
mock.patch.object(triage, "CONFIG_PATH", "/nonexistent/config.json"):
|
|
77
|
+
self.assertFalse(triage.laya_preferred())
|
|
78
|
+
|
|
79
|
+
def test_main_exits_2_and_says_why(self):
|
|
80
|
+
with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as f:
|
|
81
|
+
f.write("refund please\n\nthis is broken\n")
|
|
82
|
+
err = io.StringIO()
|
|
83
|
+
with mock.patch.dict(os.environ, {"LUCKIEST_LAYA": "0"}), mock.patch("sys.stderr", err):
|
|
84
|
+
self.assertEqual(triage.main([f.name]), 2)
|
|
85
|
+
self.assertIn("Tag messages by hand", err.getvalue())
|
|
86
|
+
os.unlink(f.name)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class Input(unittest.TestCase):
|
|
90
|
+
def test_csv_column_and_blank_lines(self):
|
|
91
|
+
with tempfile.NamedTemporaryFile("w", suffix=".csv", delete=False, newline="") as f:
|
|
92
|
+
f.write('id,body\n1,"line one\nstill one"\n2,\n3,second\n')
|
|
93
|
+
self.assertEqual(triage.read_messages(f.name, "body"), ["line one still one", "second"])
|
|
94
|
+
with self.assertRaises(SystemExit):
|
|
95
|
+
triage.read_messages(f.name, "missing")
|
|
96
|
+
os.unlink(f.name)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
if __name__ == "__main__":
|
|
100
|
+
unittest.main()
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Tag customer messages (reviews, tickets, survey answers) with Laya, locally.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
python3 triage.py messages.txt # one message per line
|
|
6
|
+
python3 triage.py reviews.csv --column body # one CSV column
|
|
7
|
+
cat messages.txt | python3 triage.py
|
|
8
|
+
|
|
9
|
+
Optional. It runs only when the user prefers Laya ({"laya": true} in
|
|
10
|
+
~/.luckiest/config.json, or LUCKIEST_LAYA=1) and Laya is installed. Otherwise
|
|
11
|
+
it exits 2 with one line on stderr, and the skill tags messages by hand as
|
|
12
|
+
before. Exit 1 means bad input.
|
|
13
|
+
|
|
14
|
+
Output: a JSON summary on stdout: intent counts, the share of messages that are
|
|
15
|
+
urgent, ask for a refund, or hint at leaving, average frustration (0 calm to 3
|
|
16
|
+
very angry), and the most flagged messages to read first. Messages Laya is
|
|
17
|
+
unsure about are counted in `unsure` and left out of the numbers.
|
|
18
|
+
|
|
19
|
+
Same copy in luckiest-customer-research and luckiest-churn-prevention.
|
|
20
|
+
"""
|
|
21
|
+
import csv
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import shutil
|
|
25
|
+
import subprocess
|
|
26
|
+
import sys
|
|
27
|
+
|
|
28
|
+
CONFIDENCE_FLOOR = 0.4
|
|
29
|
+
FLAGGED_LIMIT = 20
|
|
30
|
+
CONFIG_PATH = os.path.expanduser("~/.luckiest/config.json")
|
|
31
|
+
INTENTS = ["refund", "technical_help", "billing_question", "information", "cancellation", "other"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Unavailable(Exception):
|
|
35
|
+
"""Laya is off, missing, or failed. The skill falls back to tagging by hand."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def laya_preferred():
|
|
39
|
+
env = os.environ.get("LUCKIEST_LAYA")
|
|
40
|
+
if env:
|
|
41
|
+
return env.strip().lower() in ("1", "true", "yes", "on")
|
|
42
|
+
try:
|
|
43
|
+
with open(CONFIG_PATH) as f:
|
|
44
|
+
return json.load(f).get("laya") is True
|
|
45
|
+
except (OSError, ValueError, AttributeError):
|
|
46
|
+
return False
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def laya_bin():
|
|
50
|
+
venv = os.path.expanduser("~/.venvs/laya/bin/laya")
|
|
51
|
+
return shutil.which("laya") or (venv if os.access(venv, os.X_OK) else None)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def read_messages(path=None, column=None):
|
|
55
|
+
f = open(path, newline="", encoding="utf-8") if path else sys.stdin
|
|
56
|
+
with f:
|
|
57
|
+
if column:
|
|
58
|
+
reader = csv.DictReader(f)
|
|
59
|
+
if column not in (reader.fieldnames or []):
|
|
60
|
+
sys.exit(f"column {column!r} not found, have: {', '.join(reader.fieldnames or [])}")
|
|
61
|
+
rows = [r[column] or "" for r in reader]
|
|
62
|
+
else:
|
|
63
|
+
rows = f.read().splitlines()
|
|
64
|
+
return [" ".join(r.split()) for r in rows if r.strip()]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def laya_answers(messages):
|
|
68
|
+
"""One Laya batch call for all messages. Raises Unavailable on any failure."""
|
|
69
|
+
if not laya_preferred():
|
|
70
|
+
raise Unavailable("laya not preferred (set {\"laya\": true} in ~/.luckiest/config.json)")
|
|
71
|
+
exe = laya_bin()
|
|
72
|
+
if not exe:
|
|
73
|
+
raise Unavailable("laya not installed")
|
|
74
|
+
try:
|
|
75
|
+
run = subprocess.run([exe, "--preset", "triage", "--json", "--batch", "-"],
|
|
76
|
+
input="\n".join(messages) + "\n", capture_output=True, text=True,
|
|
77
|
+
timeout=60 + len(messages) // 2)
|
|
78
|
+
if run.returncode != 0:
|
|
79
|
+
raise Unavailable(f"laya failed: {run.stderr.strip()[:200]}")
|
|
80
|
+
answers = [json.loads(l)["answers"] for l in run.stdout.splitlines() if l.strip()]
|
|
81
|
+
except (OSError, subprocess.TimeoutExpired, ValueError, KeyError, TypeError) as e:
|
|
82
|
+
raise Unavailable(f"laya failed: {type(e).__name__}")
|
|
83
|
+
if len(answers) != len(messages):
|
|
84
|
+
raise Unavailable("laya returned the wrong number of rows")
|
|
85
|
+
return answers
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def tag(ans):
|
|
89
|
+
"""One message's tags, or None when Laya is unsure of its intent."""
|
|
90
|
+
try:
|
|
91
|
+
intent = ans["intent"]
|
|
92
|
+
if intent["answer_confidence"] < CONFIDENCE_FLOOR:
|
|
93
|
+
return None
|
|
94
|
+
return {
|
|
95
|
+
"intent": intent["choice"],
|
|
96
|
+
"frustration": round(float(ans["frustration"]["score"]), 1),
|
|
97
|
+
"urgent": ans["is_urgent"]["noul"] >= 0.5,
|
|
98
|
+
"refund": ans["refund_requested"]["noul"] >= 0.5,
|
|
99
|
+
"churn_risk": ans["churn_risk"]["noul"] >= 0.5,
|
|
100
|
+
}
|
|
101
|
+
except (KeyError, TypeError, ValueError):
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def summarize(messages, answers):
|
|
106
|
+
tags = [(i, tag(a)) for i, a in enumerate(answers)]
|
|
107
|
+
rows = [(i, t) for i, t in tags if t]
|
|
108
|
+
n = len(rows)
|
|
109
|
+
pct = lambda key: round(100 * sum(t[key] for _, t in rows) / n) if n else 0
|
|
110
|
+
intents = {k: 0 for k in INTENTS}
|
|
111
|
+
for _, t in rows:
|
|
112
|
+
intents[t["intent"]] = intents.get(t["intent"], 0) + 1
|
|
113
|
+
# Read first: angriest, then leaving, refund, urgent.
|
|
114
|
+
weight = lambda t: t["frustration"] + 2 * t["churn_risk"] + t["refund"] + t["urgent"]
|
|
115
|
+
flagged = sorted((r for r in rows if weight(r[1]) >= 2), key=lambda r: -weight(r[1]))
|
|
116
|
+
return {
|
|
117
|
+
"engine": "laya",
|
|
118
|
+
"messages": len(messages),
|
|
119
|
+
"tagged": n,
|
|
120
|
+
"unsure": len(messages) - n,
|
|
121
|
+
"intent": intents,
|
|
122
|
+
"urgent_pct": pct("urgent"),
|
|
123
|
+
"refund_pct": pct("refund"),
|
|
124
|
+
"churn_risk_pct": pct("churn_risk"),
|
|
125
|
+
"avg_frustration": round(sum(t["frustration"] for _, t in rows) / n, 2) if n else 0,
|
|
126
|
+
"flagged": [dict(t, row=i + 1, text=messages[i][:240]) for i, t in flagged[:FLAGGED_LIMIT]],
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def main(argv):
|
|
131
|
+
column = None
|
|
132
|
+
if "--column" in argv:
|
|
133
|
+
i = argv.index("--column")
|
|
134
|
+
if i + 1 >= len(argv):
|
|
135
|
+
sys.exit("--column needs a name")
|
|
136
|
+
column = argv[i + 1]
|
|
137
|
+
del argv[i:i + 2]
|
|
138
|
+
messages = read_messages(argv[0] if argv else None, column)
|
|
139
|
+
if not messages:
|
|
140
|
+
sys.exit("no messages found")
|
|
141
|
+
try:
|
|
142
|
+
answers = laya_answers(messages)
|
|
143
|
+
except Unavailable as e:
|
|
144
|
+
print(f"laya unavailable: {e}. Tag messages by hand.", file=sys.stderr)
|
|
145
|
+
return 2
|
|
146
|
+
print(json.dumps(summarize(messages, answers), indent=2))
|
|
147
|
+
return 0
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
if __name__ == "__main__":
|
|
151
|
+
sys.exit(main(sys.argv[1:]))
|
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Changelog — luckiest-model-router
|
|
2
2
|
|
|
3
|
+
## 2.3.0 — 2026-11-01
|
|
4
|
+
Optional Laya engine for model mode.
|
|
5
|
+
|
|
6
|
+
- Users who prefer Laya set `{"model_router": "laya"}` in `~/.luckiest/config.json`
|
|
7
|
+
or `LUCKIEST_MODEL_ROUTER=laya`. One `laya --preset router --batch` call per plan.
|
|
8
|
+
- Rules stay in charge: Laya picks the tier only when the rules are unsure, and
|
|
9
|
+
otherwise moves it one step when at least 60% sure. Light floor still applies.
|
|
10
|
+
- Batch items take an optional `detail` (the "done means" line) for Laya.
|
|
11
|
+
- Missing, slow (60s + 0.5s per task), failing or unsure Laya leaves the rules' pick and adds a warning.
|
|
12
|
+
|
|
3
13
|
## 2.2.0 — 2026-10-16
|
|
4
14
|
Model ids refreshed (issue #198), matching `server/mcp/modelHint.js`.
|
|
5
15
|
|
|
@@ -3,7 +3,7 @@ name: model-router
|
|
|
3
3
|
description: Use when the user asks which agent or which model should handle a task. Two modes. Agent mode detects which of Claude, Codex, Cursor, Conductor, or Factory is installed and picks the best fit for the prompt. Model mode tags each task with a Haiku, Sonnet, or Opus tier by task type, so subagents run on the smallest model that fits. Triggers include "which tool should I use", "route this", "best model for this task", "haiku or opus", "which model for each task", "tag tasks with models", and /luckiest plan drafting a task list. Use it whenever someone is choosing between coding agents or model sizes, even if they don't name one.
|
|
4
4
|
license: See ATTRIBUTION.md
|
|
5
5
|
metadata:
|
|
6
|
-
version: "2.
|
|
6
|
+
version: "2.3.0"
|
|
7
7
|
listing_id: luckiest-model-router
|
|
8
8
|
author: luckiest
|
|
9
9
|
---
|
|
@@ -21,9 +21,9 @@ Rules for both modes:
|
|
|
21
21
|
|
|
22
22
|
## Staying current
|
|
23
23
|
|
|
24
|
-
On activation, call the Luckiest MCP check_updates tool with { listingId: "641e0cd7-0695-4caf-b97a-f7a0b3f872ec", installedSemver: "2.
|
|
24
|
+
On activation, call the Luckiest MCP check_updates tool with { listingId: "641e0cd7-0695-4caf-b97a-f7a0b3f872ec", installedSemver: "2.3.0" }. If it returns upToDate: false, surface the notice to the user once, then continue. Do nothing further if upToDate: true. Never block on this check. If the tool is unavailable, proceed.
|
|
25
25
|
|
|
26
|
-
When the skill's work is done, call the Luckiest MCP report_usage tool once with { listing_id: "luckiest-model-router", skill_version: "2.
|
|
26
|
+
When the skill's work is done, call the Luckiest MCP report_usage tool once with { listing_id: "luckiest-model-router", skill_version: "2.3.0", matched: true, success: <true if the skill completed, false otherwise> }. Send metadata only, never prompt text. Never block on it. If the tool is unavailable, skip silently.
|
|
27
27
|
|
|
28
28
|
## Model mode
|
|
29
29
|
|
|
@@ -63,6 +63,15 @@ Use this mode when the question is about model size, or when /luckiest plan is t
|
|
|
63
63
|
4. Treat the script as a starting point. When the title undersells the task, raise the tier and say why in the reason. Only lower a tier when the task is clearly trivial.
|
|
64
64
|
5. Model IDs: light `claude-haiku-5-5`, standard `claude-sonnet-5-5`, heavy `claude-opus-5-5`. Setting `LUCKIEST_MODEL_PROVIDER=openai` or `google` switches to that provider's table, which matches the server. When showing tiers to the user, say **Haiku**, **Sonnet** or **Opus**. When a subagent runs the task, always pass its model explicitly. Claude Code subagents inherit the session model when none is set, and that is Opus by default.
|
|
65
65
|
|
|
66
|
+
### Laya engine (optional)
|
|
67
|
+
|
|
68
|
+
Users who have [Laya](https://huggingface.co/convaiinnovations/laya) installed can prefer it. Turn it on with `{"laya": true}` in `~/.luckiest/config.json` (the shared switch other skills read too) or `{"model_router": "laya"}` for routing only. `LUCKIEST_LAYA=1` and `LUCKIEST_MODEL_ROUTER=laya` do the same from the environment. `LUCKIEST_MODEL_ROUTER=rules` turns Laya off for routing and wins over everything else. The script finds `laya` on PATH or at `~/.venvs/laya/bin/laya` and makes one batch call for the whole plan.
|
|
69
|
+
|
|
70
|
+
- The rules above still decide. Laya picks the tier only for tasks the rules mark unsure, and otherwise moves the rules' tier by one step when it is at least 60% sure. The light floor for tool-heavy work still applies.
|
|
71
|
+
- Give each batch item an optional `detail` (for example the task's "done means" line). Laya tells tasks apart much better with a sentence than with a bare title.
|
|
72
|
+
- A reason that contains `laya:` means Laya picked or nudged that tier.
|
|
73
|
+
- If Laya is missing, slow (over 60 seconds plus half a second per task), fails, or is unsure, the rules' pick stands and the output has a warning.
|
|
74
|
+
|
|
66
75
|
### Escalation during execution
|
|
67
76
|
|
|
68
77
|
When a subagent's result fails its "done means" check twice on the same task, re-run the task one tier higher from the start and tell the user in one line. Do not hand a half-done run to the bigger model, because switching models mid-task rewrites most of the actions that follow. Tasks that depend on it and have not started yet move up to at least the new tier. Heavy is the ceiling. If a heavy run still fails, stop and hand the task back to the user.
|
|
@@ -7,7 +7,8 @@ Usage:
|
|
|
7
7
|
python3 route.py --list # just show what's installed
|
|
8
8
|
python3 route.py --model "task title" [--prefer quality|balanced|cheap]
|
|
9
9
|
python3 route.py --model --batch tasks.json # or JSON on stdin
|
|
10
|
-
tasks.json: [{"title": ..., "skill": ..., "depends_on": [0]}]
|
|
10
|
+
tasks.json: [{"title": ..., "skill": ..., "depends_on": [0], "detail": ...}]
|
|
11
|
+
detail (optional) is only read by the Laya engine.
|
|
11
12
|
|
|
12
13
|
--model picks a model tier (light/standard/heavy) per task. The rules are
|
|
13
14
|
ported from Weave Router (Apache 2.0): route each task on its own, classify
|
|
@@ -25,6 +26,7 @@ import json
|
|
|
25
26
|
import os
|
|
26
27
|
import re
|
|
27
28
|
import shutil
|
|
29
|
+
import subprocess
|
|
28
30
|
import sys
|
|
29
31
|
|
|
30
32
|
# How each tool announces itself on the system. First hit wins.
|
|
@@ -165,7 +167,87 @@ def _shift(tier, n):
|
|
|
165
167
|
return TIERS[max(0, min(len(TIERS) - 1, TIERS.index(tier) + n))]
|
|
166
168
|
|
|
167
169
|
|
|
168
|
-
|
|
170
|
+
# ---- Laya engine (opt-in) -------------------------------------------------
|
|
171
|
+
# Users who prefer Laya set {"laya": true} (any skill) or {"model_router": "laya"}
|
|
172
|
+
# in ~/.luckiest/config.json, or LUCKIEST_LAYA=1 / LUCKIEST_MODEL_ROUTER=laya. Laya's router preset scores difficulty 0-3 and
|
|
173
|
+
# says whether the task needs tools or is sensitive; laya_tier maps that to a
|
|
174
|
+
# tier. The rules stay in charge: Laya decides only when they are unsure, and
|
|
175
|
+
# otherwise moves their tier one step when it is confident. On short task text
|
|
176
|
+
# Laya's scores bunch together, so it is an assistant, not the judge.
|
|
177
|
+
# Any Laya failure or low confidence leaves the rules' pick as is.
|
|
178
|
+
LAYA_CONFIDENCE_FLOOR = 0.4
|
|
179
|
+
LAYA_NUDGE_CONFIDENCE = 0.6
|
|
180
|
+
LAYA_TIMEOUT = 60 # cold start alone can pass 30s; plus half a second per task
|
|
181
|
+
CONFIG_PATH = os.path.expanduser("~/.luckiest/config.json")
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def laya_preferred():
|
|
185
|
+
# LUCKIEST_MODEL_ROUTER (laya or rules) wins for routing; LUCKIEST_LAYA and
|
|
186
|
+
# {"laya": true} are the shared "use Laya wherever skills support it" switch.
|
|
187
|
+
env = os.environ.get("LUCKIEST_MODEL_ROUTER")
|
|
188
|
+
if env:
|
|
189
|
+
return env.strip().lower() == "laya"
|
|
190
|
+
env = os.environ.get("LUCKIEST_LAYA")
|
|
191
|
+
if env:
|
|
192
|
+
return env.strip().lower() in ("1", "true", "yes", "on")
|
|
193
|
+
try:
|
|
194
|
+
with open(CONFIG_PATH) as f:
|
|
195
|
+
cfg = json.load(f)
|
|
196
|
+
return cfg.get("model_router") == "laya" or cfg.get("laya") is True
|
|
197
|
+
except (OSError, ValueError, AttributeError):
|
|
198
|
+
return False
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def laya_bin():
|
|
202
|
+
venv = os.path.expanduser("~/.venvs/laya/bin/laya")
|
|
203
|
+
return shutil.which("laya") or (venv if os.access(venv, os.X_OK) else None)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def laya_answers(titles):
|
|
207
|
+
"""One Laya batch call for all titles. Returns a list of answer dicts, or None."""
|
|
208
|
+
exe = laya_bin()
|
|
209
|
+
if not exe:
|
|
210
|
+
return None
|
|
211
|
+
lines = "\n".join(" ".join(t.split()) or "-" for t in titles) + "\n"
|
|
212
|
+
try:
|
|
213
|
+
run = subprocess.run([exe, "--preset", "router", "--json", "--batch", "-"],
|
|
214
|
+
input=lines, capture_output=True, text=True, timeout=LAYA_TIMEOUT + len(titles) // 2)
|
|
215
|
+
if run.returncode != 0:
|
|
216
|
+
return None
|
|
217
|
+
rows = [json.loads(l) for l in run.stdout.splitlines() if l.strip()]
|
|
218
|
+
answers = [r["answers"] for r in rows]
|
|
219
|
+
except (OSError, subprocess.TimeoutExpired, ValueError, KeyError, TypeError):
|
|
220
|
+
return None
|
|
221
|
+
return answers if len(answers) == len(titles) else None
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def laya_tier(ans):
|
|
225
|
+
"""Map Laya router answers to (tier, reason, confidence), or None when Laya is unsure."""
|
|
226
|
+
try:
|
|
227
|
+
d = ans["difficulty"]
|
|
228
|
+
confidence = float(d["answer_confidence"])
|
|
229
|
+
if confidence < LAYA_CONFIDENCE_FLOOR:
|
|
230
|
+
return None
|
|
231
|
+
score = float(d["score"])
|
|
232
|
+
tools = ans["needs_tools"]["noul"] >= 0.5
|
|
233
|
+
sensitive = ans["is_sensitive"]["noul"] >= 0.5
|
|
234
|
+
except (KeyError, TypeError, ValueError):
|
|
235
|
+
return None
|
|
236
|
+
# ponytail: cutoffs tuned on a handful of tasks; Laya's expected score bunches
|
|
237
|
+
# around 1-2, so "hard" starts at 1.8. Retune against real plans if tiers drift.
|
|
238
|
+
if score >= 1.8 or (sensitive and score >= 1.5):
|
|
239
|
+
tier = "heavy"
|
|
240
|
+
elif score < 1.2 and not tools:
|
|
241
|
+
tier = "light"
|
|
242
|
+
else:
|
|
243
|
+
tier = "standard"
|
|
244
|
+
reason = f"laya: difficulty {score:.1f}"
|
|
245
|
+
reason += ", sensitive" if sensitive else ""
|
|
246
|
+
reason += ", needs tools" if tools else ""
|
|
247
|
+
return tier, reason, confidence
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def route_task(title, skill=None, prefer="balanced", laya=None):
|
|
169
251
|
text = f"{title} {skill or ''}"
|
|
170
252
|
task_type, tier, hits = None, None, []
|
|
171
253
|
for name, base, pattern in TASK_TYPES:
|
|
@@ -173,27 +255,35 @@ def route_task(title, skill=None, prefer="balanced"):
|
|
|
173
255
|
if m:
|
|
174
256
|
task_type, tier, hits = name, base, [m.group(0)]
|
|
175
257
|
break
|
|
176
|
-
if
|
|
258
|
+
picked = laya_tier(laya) if laya else None
|
|
259
|
+
if task_type is None and not picked:
|
|
177
260
|
# Weave rule: never fail open to the cheap model. Unsure means standard.
|
|
178
261
|
return {"tier": "standard", "model": MODELS["standard"], "task_type": "unsure",
|
|
179
262
|
"reason": "unsure: no task-type signal, defaulting to standard", "signals": []}
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
263
|
+
if task_type is None:
|
|
264
|
+
tier, reason = picked[0], picked[1] # rules unsure: Laya decides
|
|
265
|
+
else:
|
|
266
|
+
reason = f"{task_type} task"
|
|
267
|
+
up = re.search(UPSHIFT, text, re.I)
|
|
268
|
+
if up:
|
|
269
|
+
tier = _shift(tier, 1)
|
|
270
|
+
hits.append(up.group(0))
|
|
271
|
+
reason += f", upshift for '{up.group(0)}'"
|
|
272
|
+
elif "```" in title or len(title.split()) > 60:
|
|
273
|
+
# Same rule as the server fallback: pasted code or a long spec is heavier work.
|
|
274
|
+
tier = _shift(tier, 1)
|
|
275
|
+
reason += ", upshift for long or code-bearing task"
|
|
276
|
+
if picked and picked[2] >= LAYA_NUDGE_CONFIDENCE and picked[0] != tier:
|
|
277
|
+
step = 1 if TIERS.index(picked[0]) > TIERS.index(tier) else -1
|
|
278
|
+
tier = _shift(tier, step)
|
|
279
|
+
reason += f", {picked[1]}, nudged {'up' if step > 0 else 'down'}"
|
|
190
280
|
if PREFER_SHIFT.get(prefer, 0):
|
|
191
281
|
tier = _shift(tier, PREFER_SHIFT[prefer])
|
|
192
282
|
reason += f", prefer {prefer}"
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
return {"tier": tier, "model": MODELS[tier], "task_type": task_type,
|
|
283
|
+
if tier == "light" and task_type not in LIGHT_FLOOR_EXEMPT:
|
|
284
|
+
tier = "standard"
|
|
285
|
+
reason += ", held at standard (tool-heavy work loops on small models)"
|
|
286
|
+
return {"tier": tier, "model": MODELS[tier], "task_type": task_type or "unsure",
|
|
197
287
|
"reason": reason, "signals": hits}
|
|
198
288
|
|
|
199
289
|
|
|
@@ -203,9 +293,15 @@ def route_batch(tasks, prefer="balanced"):
|
|
|
203
293
|
A task that depends on another inherits the higher of the two tiers, so a
|
|
204
294
|
chain shares one model and its context, unless a later task needs more.
|
|
205
295
|
"""
|
|
206
|
-
out = [dict(route_task(t.get("title", ""), t.get("skill"), prefer), title=t.get("title", ""))
|
|
207
|
-
for t in tasks]
|
|
208
296
|
warnings = []
|
|
297
|
+
# Laya separates tasks far better with a sentence of detail than a bare title.
|
|
298
|
+
texts = [f"{t.get('title', '')} {t.get('detail') or ''}" for t in tasks]
|
|
299
|
+
answers = laya_answers(texts) if laya_preferred() else None
|
|
300
|
+
if laya_preferred() and answers is None:
|
|
301
|
+
warnings.append("laya preferred but unavailable, used built-in rules")
|
|
302
|
+
answers = answers or [None] * len(tasks)
|
|
303
|
+
out = [dict(route_task(t.get("title", ""), t.get("skill"), prefer, a), title=t.get("title", ""))
|
|
304
|
+
for t, a in zip(tasks, answers)]
|
|
209
305
|
for i, t in enumerate(tasks):
|
|
210
306
|
for dep in t.get("depends_on") or []:
|
|
211
307
|
if not (isinstance(dep, int) and 0 <= dep < i):
|
|
@@ -267,7 +363,11 @@ def model_main(args):
|
|
|
267
363
|
title = " ".join(args).strip() or (sys.stdin.read().strip() if not sys.stdin.isatty() else "")
|
|
268
364
|
if not title:
|
|
269
365
|
sys.exit("give a task title")
|
|
270
|
-
|
|
366
|
+
routed, warnings = route_batch([{"title": title}], prefer)
|
|
367
|
+
result = dict(routed[0], prefer=prefer)
|
|
368
|
+
if warnings:
|
|
369
|
+
result["warnings"] = warnings
|
|
370
|
+
print(json.dumps(result, indent=2))
|
|
271
371
|
|
|
272
372
|
|
|
273
373
|
if __name__ == "__main__":
|
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
"""Run: python3 -m unittest scripts/test_route.py (from the skill directory)."""
|
|
2
2
|
import os
|
|
3
3
|
import sys
|
|
4
|
+
import json
|
|
5
|
+
import subprocess
|
|
6
|
+
import tempfile
|
|
4
7
|
import unittest
|
|
8
|
+
from unittest import mock
|
|
5
9
|
|
|
10
|
+
os.environ["LUCKIEST_MODEL_ROUTER"] = "rules" # keep the user's own preference out of tests
|
|
6
11
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
7
12
|
import route # noqa: E402
|
|
8
13
|
|
|
@@ -49,5 +54,88 @@ class RouteTask(unittest.TestCase):
|
|
|
49
54
|
{"light": "claude-haiku-5-5", "standard": "claude-sonnet-5-5", "heavy": "claude-opus-5-5"})
|
|
50
55
|
|
|
51
56
|
|
|
57
|
+
def laya_ans(score, conf=0.9, tools=0.1, sensitive=0.1):
|
|
58
|
+
return {"difficulty": {"score": score, "answer_confidence": conf},
|
|
59
|
+
"needs_tools": {"noul": tools}, "is_sensitive": {"noul": sensitive}}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class Laya(unittest.TestCase):
|
|
63
|
+
def test_mapping(self):
|
|
64
|
+
self.assertEqual(route.laya_tier(laya_ans(2.8))[0], "heavy")
|
|
65
|
+
self.assertEqual(route.laya_tier(laya_ans(1.6, sensitive=0.9))[0], "heavy")
|
|
66
|
+
self.assertEqual(route.laya_tier(laya_ans(1.6))[0], "standard")
|
|
67
|
+
self.assertEqual(route.laya_tier(laya_ans(0.6, tools=0.9))[0], "standard")
|
|
68
|
+
self.assertEqual(route.laya_tier(laya_ans(0.6))[0], "light")
|
|
69
|
+
self.assertIsNone(route.laya_tier(laya_ans(2.8, conf=0.2))) # unsure: rules decide
|
|
70
|
+
|
|
71
|
+
def test_laya_decides_only_when_rules_unsure(self):
|
|
72
|
+
r = route.route_task("polish the thing", laya=laya_ans(2.8))
|
|
73
|
+
self.assertEqual(r["tier"], "heavy")
|
|
74
|
+
self.assertTrue(r["reason"].startswith("laya:"))
|
|
75
|
+
|
|
76
|
+
def test_laya_nudges_one_step_when_confident(self):
|
|
77
|
+
r = route.route_task("rename the settings label", laya=laya_ans(2.8))
|
|
78
|
+
self.assertEqual(r["tier"], "standard") # light -> standard, never two steps
|
|
79
|
+
self.assertIn("laya:", r["reason"])
|
|
80
|
+
r = route.route_task("draft the launch email", laya=laya_ans(0.4))
|
|
81
|
+
self.assertEqual(r["tier"], "light")
|
|
82
|
+
# under the nudge confidence, the rules' pick stands
|
|
83
|
+
r = route.route_task("draft the launch email", laya=laya_ans(2.8, conf=0.5))
|
|
84
|
+
self.assertEqual((r["tier"], r["reason"]), ("standard", "content task"))
|
|
85
|
+
|
|
86
|
+
def test_laya_keeps_floor(self):
|
|
87
|
+
# implement work never drops to light, even when Laya calls it easy
|
|
88
|
+
self.assertEqual(route.route_task("add a pricing FAQ", laya=laya_ans(0.3))["tier"], "standard")
|
|
89
|
+
# low confidence falls back to the rules
|
|
90
|
+
r = route.route_task("design the billing schema", laya=laya_ans(0.3, conf=0.1))
|
|
91
|
+
self.assertEqual(r["reason"], "design task")
|
|
92
|
+
|
|
93
|
+
def run_batch(self, run=None, side_effect=None, exe="/bin/laya"):
|
|
94
|
+
with mock.patch.dict(os.environ, {"LUCKIEST_MODEL_ROUTER": "laya"}), \
|
|
95
|
+
mock.patch.object(route, "laya_bin", return_value=exe), \
|
|
96
|
+
mock.patch.object(route.subprocess, "run", return_value=run, side_effect=side_effect) as m:
|
|
97
|
+
out, warnings = route.route_batch([{"title": "draft the email"}, {"title": "add the route"}])
|
|
98
|
+
return out, warnings, m
|
|
99
|
+
|
|
100
|
+
def test_batch_one_laya_call(self):
|
|
101
|
+
stdout = "\n".join(json.dumps({"answers": a}) for a in (laya_ans(0.4), laya_ans(2.9)))
|
|
102
|
+
out, warnings, m = self.run_batch(subprocess.CompletedProcess([], 0, stdout, ""))
|
|
103
|
+
self.assertEqual(m.call_count, 1)
|
|
104
|
+
self.assertEqual([t["tier"] for t in out], ["light", "heavy"])
|
|
105
|
+
self.assertEqual(warnings, [])
|
|
106
|
+
|
|
107
|
+
def test_batch_fallbacks(self):
|
|
108
|
+
cases = [
|
|
109
|
+
dict(exe=None), # not installed
|
|
110
|
+
dict(side_effect=subprocess.TimeoutExpired("laya", 30)),
|
|
111
|
+
dict(run=subprocess.CompletedProcess([], 1, "", "boom")),
|
|
112
|
+
dict(run=subprocess.CompletedProcess([], 0, "not json", "")),
|
|
113
|
+
]
|
|
114
|
+
for kw in cases:
|
|
115
|
+
out, warnings, _ = self.run_batch(**kw)
|
|
116
|
+
self.assertEqual([t["reason"] for t in out], ["content task", "implement task"], kw)
|
|
117
|
+
self.assertEqual(len(warnings), 1, kw)
|
|
118
|
+
|
|
119
|
+
def test_shared_switch(self):
|
|
120
|
+
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
|
|
121
|
+
f.write('{"laya": true}')
|
|
122
|
+
with mock.patch.object(route, "CONFIG_PATH", f.name):
|
|
123
|
+
with mock.patch.dict(os.environ, {"LUCKIEST_MODEL_ROUTER": "", "LUCKIEST_LAYA": ""}):
|
|
124
|
+
self.assertTrue(route.laya_preferred()) # shared config key
|
|
125
|
+
with mock.patch.dict(os.environ, {"LUCKIEST_MODEL_ROUTER": "rules"}):
|
|
126
|
+
self.assertFalse(route.laya_preferred()) # routing-only off switch wins
|
|
127
|
+
os.unlink(f.name)
|
|
128
|
+
with mock.patch.dict(os.environ, {"LUCKIEST_MODEL_ROUTER": "", "LUCKIEST_LAYA": "1"}):
|
|
129
|
+
self.assertTrue(route.laya_preferred())
|
|
130
|
+
|
|
131
|
+
def test_off_by_default(self):
|
|
132
|
+
with mock.patch.dict(os.environ, {"LUCKIEST_MODEL_ROUTER": "", "LUCKIEST_LAYA": ""}), \
|
|
133
|
+
mock.patch.object(route, "CONFIG_PATH", "/nonexistent/config.json"), \
|
|
134
|
+
mock.patch.object(route.subprocess, "run") as m:
|
|
135
|
+
self.assertFalse(route.laya_preferred())
|
|
136
|
+
route.route_batch([{"title": "add the route"}])
|
|
137
|
+
m.assert_not_called()
|
|
138
|
+
|
|
139
|
+
|
|
52
140
|
if __name__ == "__main__":
|
|
53
141
|
unittest.main()
|