@letterstory/cli 0.73.0 → 0.74.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli.mjs +16 -3
- package/lib/commands/admin.mjs +302 -41
- package/package.json +1 -1
package/lib/cli.mjs
CHANGED
|
@@ -56,7 +56,7 @@ import {
|
|
|
56
56
|
} from "./commands.mjs";
|
|
57
57
|
|
|
58
58
|
// Keep in sync with cli/package.json.
|
|
59
|
-
export const VERSION = "0.
|
|
59
|
+
export const VERSION = "0.74.0";
|
|
60
60
|
|
|
61
61
|
// Flags that never take a value. Listing them explicitly means `deploy get --json <id>`
|
|
62
62
|
// can't accidentally swallow the id as --json's value.
|
|
@@ -581,6 +581,18 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
|
|
|
581
581
|
session (a trial per arm), linked to its
|
|
582
582
|
ledger entry. Written by \`npm run lab --
|
|
583
583
|
publish\` in the kernels repo.
|
|
584
|
+
admin spend summary [--range day|week|month|year | --from D --to D] [--grain day|week|month]
|
|
585
|
+
| entries [--from D] [--to D] [--vendor V] [--area A] [--source invoice|csv_import|api|manual]
|
|
586
|
+
| add --vendor V --amount N --from D [--to D] --area A --category C [--basis billed|estimated] …
|
|
587
|
+
| import <file.csv> [--vendor Concentrate] [--area A] [--keep-ledger-key] [--dry-run]
|
|
588
|
+
| remove <entry-id> | services | service <slug> [--owner …] [--monthly N|none] [--renewal D]
|
|
589
|
+
Where Letter's money goes (spending.letter.dev):
|
|
590
|
+
totals, run rate, area/vendor/provider/model/
|
|
591
|
+
key/operation/org breakdowns, every number's
|
|
592
|
+
source and basis, and the data gaps. Entries
|
|
593
|
+
cover what the app ledger cannot see; a CSV
|
|
594
|
+
import sums Concentrate's log export.
|
|
595
|
+
Needs spend:read (reads) / spend:write (writes).
|
|
584
596
|
admin crm import <file.csv> [--source <name>] [--tag "…"] [--dry-run]
|
|
585
597
|
| capture [shot.png …] [--text "…" | --text-file f] [--context "this is for acme.com"]
|
|
586
598
|
[--account <id>] [--dry-run] screenshots / a thread / a note → a lead
|
|
@@ -639,8 +651,9 @@ Admin (STAFF key — --staff-key / LETTERSTORY_STAFF_API_KEY; acts before an org
|
|
|
639
651
|
candidate, compare, comparisons, picks, pick,
|
|
640
652
|
ledger, reading-list, finalists,
|
|
641
653
|
finalists-queue, pair, side-by-side,
|
|
642
|
-
|
|
643
|
-
|
|
654
|
+
checklist, checklist-add, checklist-set,
|
|
655
|
+
checklist-labels, feedback, feedback-add,
|
|
656
|
+
feedback-file, feedback-reject, promote,
|
|
644
657
|
sessions, session-new, propose, trial-add,
|
|
645
658
|
trial-run, trial-save, the Write input bank
|
|
646
659
|
(inputs, input, inputs-harvest, input-save),
|
package/lib/commands/admin.mjs
CHANGED
|
@@ -8,10 +8,12 @@
|
|
|
8
8
|
// admin tuning-context [--q "<question>"] (tuning:experiment)
|
|
9
9
|
// admin tuning-import <brief-results.json> [--experiment <id>] (tuning:experiment)
|
|
10
10
|
// admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|criteria|promote … (tuning:experiment)
|
|
11
|
-
// admin tuning
|
|
12
|
-
// (
|
|
11
|
+
// admin tuning checklist|checklist-add|checklist-set|checklist-labels|feedback|feedback-add|feedback-file|feedback-reject … (tuning:experiment)
|
|
12
|
+
// (feedback-add --fix "…" files the reader's rewrite; the diff is classified and a disposition suggested.
|
|
13
|
+
// The old names — evals, eval-add, eval-set, proposals, propose-eval, capture-eval, reject-eval — still work.)
|
|
13
14
|
// admin evals scoreboard|cases|runs|run|triage|guard|retire … (fleet:read / tuning:experiment)
|
|
14
15
|
// admin crm import|capture|signals|record|signal|accounts|account|score|slack-channels|link-slack|white-glove|set-white-glove|churn|record-churn|undo-churn|onboard|onboarding|retry-onboarding|onboard-plan|payment-link … (crm:read / crm:write)
|
|
16
|
+
// admin spend summary|entries|add|import|remove|services|service … (spend:read / spend:write)
|
|
15
17
|
// admin replays [--org <id>] [--target <id>] [--status …] [--limit n] (fleet:read)
|
|
16
18
|
// admin replay --org <id> [<response-id> | --target <id>] [--surface …] (fleet:replay)
|
|
17
19
|
// admin create-org --domain acme.com [--name "Acme"] [--test] [--no-key] [--key-name N]
|
|
@@ -449,15 +451,19 @@ async function cmdAdminTuningContext(ctx) {
|
|
|
449
451
|
* admin tuning criteria the Eye Test's criteria: id, label and the question each asks of a pair
|
|
450
452
|
* admin tuning loop where the loop stands: what waits at each station, and where to go next
|
|
451
453
|
* admin tuning ledger <comparison-id> the defect ledger: per arm, what the diagnosis read named, the floor and the ceiling, the candidate's next arm; then the reader edits per arm by lever, and the schema gaps
|
|
452
|
-
* admin tuning
|
|
453
|
-
* admin tuning
|
|
454
|
-
* admin tuning
|
|
455
|
-
* admin tuning
|
|
456
|
-
* admin tuning
|
|
457
|
-
* admin tuning
|
|
458
|
-
* admin tuning
|
|
459
|
-
* admin tuning
|
|
460
|
-
* admin tuning
|
|
454
|
+
* admin tuning checklist the checklist: every defect category with its status, the picks' agreement (n, agreement, suggestion) and its labels (+/−) (alias: evals)
|
|
455
|
+
* admin tuning checklist-add --id … --label … --how … --layer … --lever … --locality local|structural --axis … [--example "span"] add a category (starts unvetted) (alias: eval-add)
|
|
456
|
+
* admin tuning checklist-set <id> [--status confirmed|rejected|unvetted] [--example "span" [--fix "…"]] confirm / reject (a person's call), or add an example (alias: eval-set)
|
|
457
|
+
* admin tuning checklist-labels <id> the labelled set of one category: positives (examples + feedback filed into it), negatives (rejected feedback suggested for it), grid overrules
|
|
458
|
+
* admin tuning feedback [--status proposed|captured|rejected] the feedback: spans readers marked in a pair's text with their words, each with its disposition (alias: proposals)
|
|
459
|
+
* admin tuning feedback-add --span "…" --comment "…" [--fix "…"] [--comparison <id> --cell <key> --side control|candidate] [--kind narrow|wide] add feedback (no --side from a blind read); --fix is the reader's rewrite, classified and a disposition suggested (alias: propose-eval)
|
|
460
|
+
* admin tuning feedback-file <id> --as checklist --id … --label … --how … --layer … --lever … --locality local|structural --axis … file it on the checklist, the span (and its fix) as the category's example (appends when the id exists) (alias: capture-eval; --as category)
|
|
461
|
+
* admin tuning feedback-file <id> --as rule file it as a rule; the rule itself is written in the kernels rule lab
|
|
462
|
+
* admin tuning feedback-file <id> --as config [--note "…"] file it as a lever this candidate should pull
|
|
463
|
+
* admin tuning feedback-file <id> --as implementation [--note "…"] file it as a schema gap no setting can express, for the kernels repo (--as lever still works)
|
|
464
|
+
* admin tuning feedback-file <id> --as material [--note "…"] file it as the material's: the input never carried it
|
|
465
|
+
* admin tuning feedback-file <id> --as note keep it; no action
|
|
466
|
+
* admin tuning feedback-reject <id> feedback that was not worth keeping after all (alias: reject-eval)
|
|
461
467
|
* admin tuning reading-list <comparison-id> [--n 10] your next blind pairs as links, to read off the screen
|
|
462
468
|
* admin tuning finalists [--limit 10] the finalists feed: done, gates clean, no craft loss, not yet decided by the picks; ranked by the objective's order
|
|
463
469
|
* admin tuning finalists-queue [--n 10] your next blind pairs round-robin across the finalists (a key reads the queue of the person who owns it; recording a pick still needs a person)
|
|
@@ -507,8 +513,8 @@ async function cmdAdminTuningContext(ctx) {
|
|
|
507
513
|
* admin tuning screens | screen <id> list / one screen: per candidate, each measure against the Base, with its noise
|
|
508
514
|
* admin tuning screen-advance <id> queue what the caps allow now (the cron does it every minute)
|
|
509
515
|
*
|
|
510
|
-
* The bake-off: a screen's versions graded item by item against the checklist
|
|
511
|
-
*
|
|
516
|
+
* The bake-off: a screen's versions graded item by item against the checklist's confirmed and
|
|
517
|
+
* unvetted categories and the craft axes commitment / economy / movement, the winner
|
|
512
518
|
* per item tallied across the inputs, and a merge that composes the best of each into one candidate.
|
|
513
519
|
* Deterministic; a person overrides a grade and confirms the merge, which is a hypothesis until the
|
|
514
520
|
* eye test confirms it.
|
|
@@ -793,6 +799,7 @@ async function cmdAdminTuning(ctx) {
|
|
|
793
799
|
})
|
|
794
800
|
);
|
|
795
801
|
}
|
|
802
|
+
case "checklist":
|
|
796
803
|
case "evals": {
|
|
797
804
|
const r = await staffRequest(ctx, "GET", `${T}/evals`);
|
|
798
805
|
if (json) return printResult(io, flags, r);
|
|
@@ -800,16 +807,29 @@ async function cmdAdminTuning(ctx) {
|
|
|
800
807
|
if (!rows.length) return io.log("(no categories)");
|
|
801
808
|
const cell = (v, w) => String(v ?? "—").padEnd(w);
|
|
802
809
|
io.log(
|
|
803
|
-
`${cell("id", 24)} ${cell("status", 10)} ${cell("layer", 10)} ${cell("lever", 30)} ${"n".padStart(4)} ${"agree".padStart(6)} ${cell("suggest", 10)} ${"
|
|
810
|
+
`${cell("id", 24)} ${cell("status", 10)} ${cell("layer", 10)} ${cell("lever", 30)} ${"n".padStart(4)} ${"agree".padStart(6)} ${cell("suggest", 10)} ${"+".padStart(3)} ${"−".padStart(3)}`
|
|
804
811
|
);
|
|
805
812
|
for (const c of rows) {
|
|
806
813
|
const v = c.vetting ?? {};
|
|
814
|
+
const l = c.labels ?? { positives: (c.examples ?? []).length, negatives: 0 };
|
|
807
815
|
io.log(
|
|
808
|
-
`${cell(c.id, 24)} ${cell(c.status, 10)} ${cell(c.layer, 10)} ${cell(c.lever || "—", 30)} ${String(v.n ?? 0).padStart(4)} ${String(v.agreement ?? "—").padStart(6)} ${cell(v.suggestion, 10)} ${String(
|
|
816
|
+
`${cell(c.id, 24)} ${cell(c.status, 10)} ${cell(c.layer, 10)} ${cell(c.lever || "—", 30)} ${String(v.n ?? 0).padStart(4)} ${String(v.agreement ?? "—").padStart(6)} ${cell(v.suggestion, 10)} ${String(l.positives).padStart(3)} ${String(l.negatives).padStart(3)}`
|
|
809
817
|
);
|
|
810
818
|
}
|
|
811
819
|
return;
|
|
812
820
|
}
|
|
821
|
+
case "checklist-labels": {
|
|
822
|
+
if (!id) throw new CliError("Usage: admin tuning checklist-labels <category-id>");
|
|
823
|
+
const r = await staffRequest(ctx, "GET", `${T}/evals/${encodeURIComponent(id)}/labels`);
|
|
824
|
+
if (json) return printResult(io, flags, r);
|
|
825
|
+
io.log(
|
|
826
|
+
`${r.category_id}: ${r.counts.positives} positive, ${r.counts.negatives} negative (${r.grid_overrules} grid overrule${r.grid_overrules === 1 ? "" : "s"})`
|
|
827
|
+
);
|
|
828
|
+
for (const x of r.positives ?? []) io.log(` + [${x.source}] "${x.span}"${x.fix ? ` → "${x.fix}"` : ""}`);
|
|
829
|
+
for (const x of r.negatives ?? []) io.log(` − [${x.source}] "${x.span}"${x.fix ? ` → "${x.fix}"` : ""}`);
|
|
830
|
+
return;
|
|
831
|
+
}
|
|
832
|
+
case "checklist-add":
|
|
813
833
|
case "eval-add": {
|
|
814
834
|
const body = {
|
|
815
835
|
id: flagStr(flags.id),
|
|
@@ -830,25 +850,26 @@ async function cmdAdminTuning(ctx) {
|
|
|
830
850
|
!["local", "structural"].includes(body.locality ?? "")
|
|
831
851
|
)
|
|
832
852
|
throw new CliError(
|
|
833
|
-
'Usage: admin tuning
|
|
853
|
+
'Usage: admin tuning checklist-add --id … --label … --how … --layer … --lever … --locality local|structural --axis … [--example "span"]'
|
|
834
854
|
);
|
|
835
855
|
if (example) body.examples = [{ span: example, ...(flags.fix ? { fix: flagStr(flags.fix) } : {}) }];
|
|
836
856
|
const r = await staffRequest(ctx, "POST", `${T}/evals`, body);
|
|
837
857
|
return json ? printResult(io, flags, r) : io.log(`added ${r.category.id} (${r.category.status})`);
|
|
838
858
|
}
|
|
859
|
+
case "checklist-set":
|
|
839
860
|
case "eval-set": {
|
|
840
861
|
const status = flagStr(flags.status);
|
|
841
862
|
const example = flagStr(flags.example);
|
|
842
863
|
if (!id || (!status && !example) || (status && !["confirmed", "rejected", "unvetted"].includes(status)))
|
|
843
864
|
throw new CliError(
|
|
844
|
-
'Usage: admin tuning
|
|
865
|
+
'Usage: admin tuning checklist-set <id> [--status confirmed|rejected|unvetted] [--example "span" [--fix "…"]]'
|
|
845
866
|
);
|
|
846
867
|
const patch = {};
|
|
847
868
|
if (status) patch.status = status;
|
|
848
869
|
if (example) {
|
|
849
870
|
// The PATCH replaces the whole list: read what is stored and append.
|
|
850
871
|
const current = (await staffRequest(ctx, "GET", `${T}/evals`)).categories?.find((c) => c.id === id);
|
|
851
|
-
if (!current) throw new CliError(`No category ${id}
|
|
872
|
+
if (!current) throw new CliError(`No category ${id} on the checklist`);
|
|
852
873
|
const fix = flagStr(flags.fix);
|
|
853
874
|
patch.examples = [...(current.examples ?? []), { span: example, ...(fix ? { fix } : {}) }];
|
|
854
875
|
}
|
|
@@ -857,27 +878,32 @@ async function cmdAdminTuning(ctx) {
|
|
|
857
878
|
? printResult(io, flags, r)
|
|
858
879
|
: io.log(`${r.category.id}: ${r.category.status}, ${(r.category.examples ?? []).length} examples`);
|
|
859
880
|
}
|
|
881
|
+
case "feedback":
|
|
860
882
|
case "proposals": {
|
|
861
883
|
const status = flagStr(flags.status);
|
|
862
884
|
if (status && !["proposed", "captured", "rejected"].includes(status))
|
|
863
|
-
throw new CliError("Usage: admin tuning
|
|
885
|
+
throw new CliError("Usage: admin tuning feedback [--status proposed|captured|rejected]");
|
|
864
886
|
const qs = status ? `?${new URLSearchParams({ status })}` : "";
|
|
865
887
|
const r = await staffRequest(ctx, "GET", `${T}/proposals${qs}`);
|
|
866
888
|
if (json) return printResult(io, flags, r);
|
|
867
889
|
const rows = r.proposals ?? [];
|
|
868
|
-
if (!rows.length) return io.log("(no
|
|
890
|
+
if (!rows.length) return io.log("(no feedback)");
|
|
869
891
|
for (const p of rows) {
|
|
870
892
|
const where = [p.cell_key, p.side].filter(Boolean).join(" / ") || "—";
|
|
893
|
+
const suggested = p.suggested_disposition
|
|
894
|
+
? `; suggested ${p.suggested_disposition}${p.suggested_category ? ` (${p.suggested_category})` : ""}`
|
|
895
|
+
: "";
|
|
871
896
|
const edit =
|
|
872
897
|
p.fix != null
|
|
873
|
-
? `\n → "${p.fix}" [${p.edit_class ?? "?"} → ${p.lever ?? "no lever: schema gap"}]`
|
|
898
|
+
? `\n → "${p.fix}" [${p.edit_class ?? "?"} → ${p.lever ?? "no lever: schema gap"}${suggested}]`
|
|
874
899
|
: "";
|
|
875
900
|
io.log(
|
|
876
|
-
`${p.id} ${p.status.padEnd(9)} ${p.kind.padEnd(
|
|
901
|
+
`${p.id} ${p.status.padEnd(9)} ${(p.disposition ?? p.kind).padEnd(14)} ${where}${p.category_id ? ` → ${p.category_id}` : ""}\n "${p.span}"${edit}${p.comment ? `\n ${p.comment}` : ""}`
|
|
877
902
|
);
|
|
878
903
|
}
|
|
879
904
|
return;
|
|
880
905
|
}
|
|
906
|
+
case "feedback-add":
|
|
881
907
|
case "propose-eval": {
|
|
882
908
|
const span = flagStr(flags.span);
|
|
883
909
|
const side = flagStr(flags.side);
|
|
@@ -888,7 +914,7 @@ async function cmdAdminTuning(ctx) {
|
|
|
888
914
|
(kind && !["narrow", "wide"].includes(kind))
|
|
889
915
|
)
|
|
890
916
|
throw new CliError(
|
|
891
|
-
'Usage: admin tuning
|
|
917
|
+
'Usage: admin tuning feedback-add --span "…" --comment "…" [--fix "…"] [--comparison <id> --cell <key> --side control|candidate] [--kind narrow|wide]'
|
|
892
918
|
);
|
|
893
919
|
const r = await staffRequest(
|
|
894
920
|
ctx,
|
|
@@ -906,17 +932,27 @@ async function cmdAdminTuning(ctx) {
|
|
|
906
932
|
);
|
|
907
933
|
if (json) return printResult(io, flags, r);
|
|
908
934
|
const p = r.proposal;
|
|
909
|
-
const
|
|
910
|
-
|
|
911
|
-
|
|
935
|
+
const suggested = p.suggested_disposition
|
|
936
|
+
? `; suggested ${p.suggested_disposition}${p.suggested_category ? ` (${p.suggested_category})` : ""}`
|
|
937
|
+
: "";
|
|
938
|
+
const edit =
|
|
939
|
+
p.fix != null ? ` — ${p.edit_class}: ${p.lever ?? "no lever maps: a schema gap"}${suggested}` : "";
|
|
940
|
+
return io.log(`added feedback ${p.id} (${p.disposition ?? p.kind})${edit}`);
|
|
941
|
+
}
|
|
942
|
+
case "feedback-file":
|
|
912
943
|
case "capture-eval": {
|
|
913
|
-
|
|
944
|
+
// The first names of checklist and implementation, still accepted.
|
|
945
|
+
const AS_ALIAS = { category: "checklist", lever: "implementation" };
|
|
946
|
+
const asGiven = flagStr(flags.as);
|
|
947
|
+
const as = AS_ALIAS[asGiven] ?? asGiven;
|
|
914
948
|
const usage =
|
|
915
|
-
'Usage: admin tuning
|
|
916
|
-
if (!id || !["
|
|
949
|
+
'Usage: admin tuning feedback-file <id> --as checklist --id … --label … --how … --layer … --lever … --locality local|structural --axis … | --as rule | --as config|implementation|material [--note "…"] | --as note';
|
|
950
|
+
if (!id || !["checklist", "rule", "config", "implementation", "material", "note"].includes(as ?? ""))
|
|
951
|
+
throw new CliError(usage);
|
|
917
952
|
let body = { as };
|
|
918
|
-
if (as === "
|
|
919
|
-
|
|
953
|
+
if (as === "config" || as === "implementation" || as === "material")
|
|
954
|
+
body = compact({ as, note: flagStr(flags.note) });
|
|
955
|
+
if (as === "checklist") {
|
|
920
956
|
const category = {
|
|
921
957
|
id: flagStr(flags.id),
|
|
922
958
|
label: flagStr(flags.label),
|
|
@@ -938,14 +974,22 @@ async function cmdAdminTuning(ctx) {
|
|
|
938
974
|
body = { as, category };
|
|
939
975
|
}
|
|
940
976
|
const r = await staffRequest(ctx, "POST", `${T}/proposals/${id}/capture`, body);
|
|
941
|
-
return
|
|
942
|
-
|
|
943
|
-
:
|
|
944
|
-
|
|
945
|
-
|
|
977
|
+
if (json) return printResult(io, flags, r);
|
|
978
|
+
const FILED = {
|
|
979
|
+
rule: "; write the rule in the kernels rule lab",
|
|
980
|
+
implementation: "; a schema gap, filed for the kernels repo",
|
|
981
|
+
config: "; a lever for this candidate",
|
|
982
|
+
material: "; the input never carried it",
|
|
983
|
+
note: "; kept, no action",
|
|
984
|
+
};
|
|
985
|
+
const d = r.proposal.disposition ?? as;
|
|
986
|
+
return io.log(
|
|
987
|
+
`filed ${r.proposal.id} as ${d}${r.category ? ` → ${r.category.id} (${(r.category.examples ?? []).length} examples)` : (FILED[d] ?? "")}`
|
|
988
|
+
);
|
|
946
989
|
}
|
|
990
|
+
case "feedback-reject":
|
|
947
991
|
case "reject-eval": {
|
|
948
|
-
if (!id) throw new CliError("Usage: admin tuning reject
|
|
992
|
+
if (!id) throw new CliError("Usage: admin tuning feedback-reject <id>");
|
|
949
993
|
const r = await staffRequest(ctx, "POST", `${T}/proposals/${id}/reject`, {});
|
|
950
994
|
return json ? printResult(io, flags, r) : io.log(`rejected ${r.proposal.id}`);
|
|
951
995
|
}
|
|
@@ -958,11 +1002,13 @@ async function cmdAdminTuning(ctx) {
|
|
|
958
1002
|
case "loop": {
|
|
959
1003
|
const r = await staffRequest(ctx, "GET", `${T}/loop`);
|
|
960
1004
|
if (json) return printResult(io, flags, r);
|
|
961
|
-
|
|
1005
|
+
const week = r.propose.candidates_week ?? "?";
|
|
1006
|
+
const progs = r.propose.programmes_running ?? "?";
|
|
1007
|
+
io.log(`propose ${r.propose.running} running, ${week} candidates this week, ${progs} programmes running`);
|
|
962
1008
|
io.log(`read ${r.read.settled_unread} settled, unread`);
|
|
963
1009
|
io.log(`eye test ${r.eye_test.finalists} finalists, ${r.eye_test.pairs_to_read} pairs to read`);
|
|
964
1010
|
io.log(
|
|
965
|
-
`
|
|
1011
|
+
`checklist ${r.evals.proposals_waiting} feedback waiting, ${r.evals.unvetted} unvetted, ${r.evals.suggested} with a suggestion${r.evals.reachable ? "" : " (the checklist could not be read)"}`
|
|
966
1012
|
);
|
|
967
1013
|
io.log(`promote ${r.promote.promotable} won their picks`);
|
|
968
1014
|
io.log(`\nnext: ${r.next}`);
|
|
@@ -2188,7 +2234,7 @@ async function cmdAdminTuning(ctx) {
|
|
|
2188
2234
|
}
|
|
2189
2235
|
default:
|
|
2190
2236
|
throw new CliError(
|
|
2191
|
-
"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|loop|ledger|
|
|
2237
|
+
"Usage: admin tuning kernels|kernel|candidate|compare|comparisons|comparison|picks|pick|loop|ledger|checklist|checklist-add|checklist-set|checklist-labels|feedback|feedback-add|feedback-file|feedback-reject|reading-list|finalists|finalists-queue|pair|bank|bank-eval|bank-draft|bank-shape|bank-pool|bank-judge|bank-score|bank-try|people|invite|revoke|assignments|assign|unassign|tasks|side-by-side|candidates|base-kernels|base-versions|sessions|session|session-new|propose|trial-add|trial-run|trial-save|promote|arenas|arena|arena-candidate|arena-dials|comparison-extend|research-compare|research-inputs|research-split|research-standing|promote-arena|shred-compare|shred-inputs|shred-split|shred-standing|shredders|shredder|shredder-candidate|promote-shredder|enhance-compare|enhance-inputs|enhance-split|enhance-standing|enhancers|enhancer|enhancer-candidate|promote-enhancer (see --help)"
|
|
2192
2238
|
);
|
|
2193
2239
|
}
|
|
2194
2240
|
}
|
|
@@ -2953,11 +2999,226 @@ async function cmdAdminReplays(ctx, sub) {
|
|
|
2953
2999
|
return 0;
|
|
2954
3000
|
}
|
|
2955
3001
|
|
|
3002
|
+
// ─── admin spend ─────────────────────────────────────────────────────────────
|
|
3003
|
+
// The headless half of spending.letter.dev: the same getSpendSummary, entries and services
|
|
3004
|
+
// the dashboard reads, through /api/superadmin/spend/*, so a background agent can watch
|
|
3005
|
+
// spend 24/7. Reads need spend:read on the staff key, writes spend:write.
|
|
3006
|
+
|
|
3007
|
+
const SPEND = "/api/superadmin/spend";
|
|
3008
|
+
const SPEND_RANGES = ["day", "week", "month", "year"];
|
|
3009
|
+
const SPEND_GRAINS = ["day", "week", "month"];
|
|
3010
|
+
const SPEND_CATEGORIES = [
|
|
3011
|
+
"ai_model",
|
|
3012
|
+
"ai_media",
|
|
3013
|
+
"search_scrape",
|
|
3014
|
+
"data_enrichment",
|
|
3015
|
+
"email_sms",
|
|
3016
|
+
"infra",
|
|
3017
|
+
"analytics",
|
|
3018
|
+
"payments",
|
|
3019
|
+
"devtools",
|
|
3020
|
+
"other",
|
|
3021
|
+
];
|
|
3022
|
+
const SPEND_BASES = ["billed", "estimated"];
|
|
3023
|
+
const SPEND_SOURCES = ["invoice", "csv_import", "api", "manual"];
|
|
3024
|
+
|
|
3025
|
+
function usd(n) {
|
|
3026
|
+
return `$${Number(n ?? 0).toLocaleString("en-US", { maximumFractionDigits: n >= 100 ? 0 : 2 })}`;
|
|
3027
|
+
}
|
|
3028
|
+
|
|
3029
|
+
function formatSpendSummary(s) {
|
|
3030
|
+
const t = s.totals;
|
|
3031
|
+
const lines = [
|
|
3032
|
+
`${s.range.from} → ${s.range.to} (${s.range.days}d): ${usd(t.total)} · ${usd(t.perDay)}/day · ~${usd(t.projectedMonth)}/mo` +
|
|
3033
|
+
(t.changePct === null ? "" : ` · ${t.changePct > 0 ? "+" : ""}${t.changePct}% vs previous`),
|
|
3034
|
+
` billed ${usd(t.billed)} · reported ${usd(t.reported)} · estimated (list price) ${usd(t.estimated)}`,
|
|
3035
|
+
"",
|
|
3036
|
+
"By area:",
|
|
3037
|
+
...s.areas.map((a) => ` ${a.label.padEnd(34)} ${usd(a.total).padStart(10)}`),
|
|
3038
|
+
"",
|
|
3039
|
+
"Top operations (letterstory):",
|
|
3040
|
+
...s.drill.slice(0, 8).map((d) => ` ${d.operation.padEnd(34)} ${usd(d.total).padStart(10)} ${d.calls} calls`),
|
|
3041
|
+
"",
|
|
3042
|
+
"Top orgs:",
|
|
3043
|
+
...s.orgs
|
|
3044
|
+
.slice(0, 10)
|
|
3045
|
+
.map(
|
|
3046
|
+
(o) =>
|
|
3047
|
+
` ${o.name.slice(0, 34).padEnd(34)} ${usd(o.total).padStart(10)}${o.internal ? ` [${o.internalReason}]` : ""}${o.paying === false && !o.internal ? " [not paying]" : ""}`
|
|
3048
|
+
),
|
|
3049
|
+
];
|
|
3050
|
+
if (s.gaps.length) lines.push("", "Not in the total:", ...s.gaps.map((g) => ` ${g.label}: ${g.reason}`));
|
|
3051
|
+
return lines.join("\n");
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
function formatSpendServices(r) {
|
|
3055
|
+
return (r.services ?? [])
|
|
3056
|
+
.map(
|
|
3057
|
+
(x) =>
|
|
3058
|
+
`${x.slug.padEnd(22)} ${x.category.padEnd(16)} ${x.live === true ? "live" : x.live === false ? "off " : " ? "} ${x.hasCostData ? usd(x.spend30d).padStart(9) + "/30d" : " no cost data"}${x.owner ? ` owner ${x.owner}` : ""}`
|
|
3059
|
+
)
|
|
3060
|
+
.join("\n");
|
|
3061
|
+
}
|
|
3062
|
+
|
|
3063
|
+
function spendEntryFromFlags(flags) {
|
|
3064
|
+
const vendor = flagStr(flags.vendor);
|
|
3065
|
+
const amount = flagNum(flags.amount);
|
|
3066
|
+
const start = flagStr(flags.from) ?? flagStr(flags.day);
|
|
3067
|
+
const end = flagStr(flags.to) ?? start;
|
|
3068
|
+
if (!vendor || amount === undefined || !start) {
|
|
3069
|
+
throw new CliError(
|
|
3070
|
+
'Usage: admin spend add --vendor Perplexity --amount 412.50 --from 2026-09-01 [--to 2026-09-30] --area lettertrace --category ai_model [--basis billed|estimated] [--source invoice|manual] [--key "…"] [--model …] [--operation …] [--service <slug>] [--ref "invoice #123"] [--note "…"] [--external-id …]'
|
|
3071
|
+
);
|
|
3072
|
+
}
|
|
3073
|
+
return compact({
|
|
3074
|
+
vendor,
|
|
3075
|
+
amount_usd: amount,
|
|
3076
|
+
period_start: start,
|
|
3077
|
+
period_end: end,
|
|
3078
|
+
area: flagStr(flags.area) ?? "other",
|
|
3079
|
+
category: oneOf(flags, "category", SPEND_CATEGORIES) ?? "other",
|
|
3080
|
+
basis: oneOf(flags, "basis", SPEND_BASES) ?? "billed",
|
|
3081
|
+
source: oneOf(flags, "source", SPEND_SOURCES) ?? "manual",
|
|
3082
|
+
key_label: flagStr(flags.key),
|
|
3083
|
+
model: flagStr(flags.model),
|
|
3084
|
+
operation: flagStr(flags.operation),
|
|
3085
|
+
service_slug: flagStr(flags.service),
|
|
3086
|
+
provider: flagStr(flags.provider),
|
|
3087
|
+
org_id: flagStr(flags.org),
|
|
3088
|
+
source_ref: flagStr(flags.ref),
|
|
3089
|
+
note: flagStr(flags.note),
|
|
3090
|
+
external_id: flagStr(flags["external-id"]),
|
|
3091
|
+
});
|
|
3092
|
+
}
|
|
3093
|
+
|
|
3094
|
+
async function cmdAdminSpend(ctx) {
|
|
3095
|
+
const { flags, positionals, io } = ctx;
|
|
3096
|
+
const sub = positionals[0];
|
|
3097
|
+
switch (sub) {
|
|
3098
|
+
case "summary": {
|
|
3099
|
+
const params = new URLSearchParams(
|
|
3100
|
+
compact({
|
|
3101
|
+
range: oneOf(flags, "range", SPEND_RANGES),
|
|
3102
|
+
from: flagStr(flags.from),
|
|
3103
|
+
to: flagStr(flags.to),
|
|
3104
|
+
grain: oneOf(flags, "grain", SPEND_GRAINS),
|
|
3105
|
+
})
|
|
3106
|
+
);
|
|
3107
|
+
const result = await staffRequest(ctx, "GET", `${SPEND}/summary${params.size ? `?${params}` : ""}`);
|
|
3108
|
+
printResult(io, flags, result, formatSpendSummary);
|
|
3109
|
+
return 0;
|
|
3110
|
+
}
|
|
3111
|
+
case "entries": {
|
|
3112
|
+
const params = new URLSearchParams(
|
|
3113
|
+
compact({
|
|
3114
|
+
from: flagStr(flags.from),
|
|
3115
|
+
to: flagStr(flags.to),
|
|
3116
|
+
vendor: flagStr(flags.vendor),
|
|
3117
|
+
area: flagStr(flags.area),
|
|
3118
|
+
source: oneOf(flags, "source", SPEND_SOURCES),
|
|
3119
|
+
limit: flagNum(flags.limit)?.toString(),
|
|
3120
|
+
})
|
|
3121
|
+
);
|
|
3122
|
+
const result = await staffRequest(ctx, "GET", `${SPEND}/entries${params.size ? `?${params}` : ""}`);
|
|
3123
|
+
printResult(io, flags, result, (r) =>
|
|
3124
|
+
(r.entries ?? [])
|
|
3125
|
+
.map(
|
|
3126
|
+
(e) =>
|
|
3127
|
+
`${e.period_start}→${e.period_end} ${usd(e.amount_usd).padStart(10)} ${e.vendor}${e.key_label ? `: ${e.key_label}` : ""} [${e.area}, ${e.basis}, ${e.source}] ${e.id}`
|
|
3128
|
+
)
|
|
3129
|
+
.join("\n")
|
|
3130
|
+
);
|
|
3131
|
+
return 0;
|
|
3132
|
+
}
|
|
3133
|
+
case "add": {
|
|
3134
|
+
const result = await staffPost(ctx, `${SPEND}/entries`, spendEntryFromFlags(flags));
|
|
3135
|
+
ok(ctx, `Recorded ${result.created} new, ${result.replaced} replaced.`);
|
|
3136
|
+
printResult(io, flags, result);
|
|
3137
|
+
return 0;
|
|
3138
|
+
}
|
|
3139
|
+
case "import": {
|
|
3140
|
+
const path = positionals[1];
|
|
3141
|
+
if (!path)
|
|
3142
|
+
throw new CliError(
|
|
3143
|
+
'Usage: admin spend import <file.csv> [--vendor Concentrate] [--area kernels] [--category ai_model] [--basis billed|estimated] [--ref "…"] [--keep-ledger-key] [--dry-run]'
|
|
3144
|
+
);
|
|
3145
|
+
const body = compact({
|
|
3146
|
+
csv: readFileSync(path, "utf8"),
|
|
3147
|
+
vendor: flagStr(flags.vendor),
|
|
3148
|
+
area: flagStr(flags.area),
|
|
3149
|
+
category: oneOf(flags, "category", SPEND_CATEGORIES),
|
|
3150
|
+
basis: oneOf(flags, "basis", SPEND_BASES),
|
|
3151
|
+
source_ref: flagStr(flags.ref) ?? `CSV import: ${path.split("/").pop()}`,
|
|
3152
|
+
exclude_ledger_key: flagBool(flags["keep-ledger-key"]) ? null : undefined,
|
|
3153
|
+
dry_run: flagBool(flags["dry-run"]) || undefined,
|
|
3154
|
+
});
|
|
3155
|
+
const result = await staffPost(ctx, `${SPEND}/import`, body);
|
|
3156
|
+
ok(
|
|
3157
|
+
ctx,
|
|
3158
|
+
`${result.used} of ${result.rows} rows → ${result.entries.length} entries, ${usd(result.total)}` +
|
|
3159
|
+
(result.days ? ` (${result.days.from} → ${result.days.to})` : "") +
|
|
3160
|
+
(result.excludedLedgerRows
|
|
3161
|
+
? `; ${result.excludedLedgerRows} BYOK rows on the ledger's key dropped`
|
|
3162
|
+
: "") +
|
|
3163
|
+
(flagBool(flags["dry-run"]) ? " (dry run, nothing written)" : "")
|
|
3164
|
+
);
|
|
3165
|
+
if (result.overlaps?.length) {
|
|
3166
|
+
ok(
|
|
3167
|
+
ctx,
|
|
3168
|
+
`Already on file for those days (may now double count): ${result.overlaps.map((o) => `${o.vendor} ${o.key_label ?? ""} ${o.id}`).join("; ")}`
|
|
3169
|
+
);
|
|
3170
|
+
}
|
|
3171
|
+
printResult(io, flags, result, () => "");
|
|
3172
|
+
return 0;
|
|
3173
|
+
}
|
|
3174
|
+
case "remove": {
|
|
3175
|
+
const id = positionals[1];
|
|
3176
|
+
if (!id) throw new CliError("Usage: admin spend remove <entry-id>");
|
|
3177
|
+
const result = await staffRequest(ctx, "DELETE", `${SPEND}/entries/${encodeURIComponent(id)}`);
|
|
3178
|
+
ok(ctx, `Removed ${id}.`);
|
|
3179
|
+
printResult(io, flags, result);
|
|
3180
|
+
return 0;
|
|
3181
|
+
}
|
|
3182
|
+
case "services": {
|
|
3183
|
+
const result = await staffRequest(ctx, "GET", `${SPEND}/services`);
|
|
3184
|
+
printResult(io, flags, result, formatSpendServices);
|
|
3185
|
+
return 0;
|
|
3186
|
+
}
|
|
3187
|
+
case "service": {
|
|
3188
|
+
const slug = positionals[1];
|
|
3189
|
+
const monthly = flags.monthly === "none" ? null : flagNum(flags.monthly);
|
|
3190
|
+
const update = compact({
|
|
3191
|
+
owner: flagStr(flags.owner),
|
|
3192
|
+
plan: flagStr(flags.plan),
|
|
3193
|
+
monthly_cost_usd: monthly,
|
|
3194
|
+
renewal_date: flagStr(flags.renewal),
|
|
3195
|
+
notes: flagStr(flags.notes),
|
|
3196
|
+
billing_source: oneOf(flags, "billing", ["api", "invoice", "unknown"]),
|
|
3197
|
+
});
|
|
3198
|
+
if (flags.monthly === "none") update.monthly_cost_usd = null;
|
|
3199
|
+
if (!slug || Object.keys(update).length === 0) {
|
|
3200
|
+
throw new CliError(
|
|
3201
|
+
'Usage: admin spend service <slug> [--owner "Steve"] [--plan "Pro"] [--monthly 20|none] [--renewal 2026-11-01] [--notes "…"] [--billing api|invoice|unknown]'
|
|
3202
|
+
);
|
|
3203
|
+
}
|
|
3204
|
+
const result = await staffRequest(ctx, "PATCH", `${SPEND}/services/${encodeURIComponent(slug)}`, update);
|
|
3205
|
+
ok(ctx, `Updated ${slug}.`);
|
|
3206
|
+
printResult(io, flags, result);
|
|
3207
|
+
return 0;
|
|
3208
|
+
}
|
|
3209
|
+
default:
|
|
3210
|
+
throw new CliError(
|
|
3211
|
+
"Usage: admin spend summary [--range day|week|month|year | --from D --to D] [--grain day|week|month] | entries [--from D] [--to D] [--vendor V] [--area A] [--source S] | add --vendor V --amount N --from D … | import <file.csv> … | remove <entry-id> | services | service <slug> --owner … --monthly …"
|
|
3212
|
+
);
|
|
3213
|
+
}
|
|
3214
|
+
}
|
|
3215
|
+
|
|
2956
3216
|
export async function cmdAdmin(ctx) {
|
|
2957
3217
|
const sub = ctx.positionals[0];
|
|
2958
3218
|
if (sub === "replays" || sub === "replay")
|
|
2959
3219
|
return cmdAdminReplays({ ...ctx, positionals: ctx.positionals.slice(1) }, sub);
|
|
2960
3220
|
if (sub === "crm") return cmdAdminCrm({ ...ctx, positionals: ctx.positionals.slice(1) });
|
|
3221
|
+
if (sub === "spend") return cmdAdminSpend({ ...ctx, positionals: ctx.positionals.slice(1) });
|
|
2961
3222
|
if (sub === "create-org") return cmdAdminCreateOrg({ ...ctx, positionals: ctx.positionals.slice(1) });
|
|
2962
3223
|
if (sub === "ask") return cmdAdminAsk({ ...ctx, positionals: ctx.positionals.slice(1) });
|
|
2963
3224
|
if (sub === "experiments") return cmdAdminExperiments({ ...ctx, positionals: ctx.positionals.slice(1) });
|
|
@@ -2966,6 +3227,6 @@ export async function cmdAdmin(ctx) {
|
|
|
2966
3227
|
if (sub === "tuning") return cmdAdminTuning({ ...ctx, positionals: ctx.positionals.slice(1) });
|
|
2967
3228
|
if (sub === "evals") return cmdAdminEvals({ ...ctx, positionals: ctx.positionals.slice(1) });
|
|
2968
3229
|
throw new CliError(
|
|
2969
|
-
'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin crm … | admin evals … | admin tuning … | admin tuning-context | admin replays … (see --help)'
|
|
3230
|
+
'Usage: admin create-org --domain <bare-domain> [--onboard] … | admin ask "<question>" | admin experiments … | admin crm … | admin spend … | admin evals … | admin tuning … | admin tuning-context | admin replays … (see --help)'
|
|
2970
3231
|
);
|
|
2971
3232
|
}
|