@awak-app/simy-cli 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/package.json +2 -2
- package/src/cli-contract.js +1 -1
- package/src/index.js +13 -0
- package/src/sqm/checkers.js +77 -17
- package/src/sqm/command.js +88 -4
- package/src/sqm/index.js +2 -1
- package/src/sqm/proof.js +10 -8
- package/src/sqm/reliability.js +94 -0
- package/src/sqm/workflow-onboarding.js +87 -0
package/README.md
CHANGED
|
@@ -9,6 +9,14 @@ npm install -g @awak-app/simy-cli
|
|
|
9
9
|
simy --daemon
|
|
10
10
|
```
|
|
11
11
|
|
|
12
|
+
When an interactive `simy --daemon` starts and the managed SIMY workflows are
|
|
13
|
+
missing or outdated, the CLI asks once before installing the Codex plugin
|
|
14
|
+
marketplace, Codex/Claude SQM skills, and PR hooks. Existing unrelated plugins
|
|
15
|
+
and hooks are preserved. Use `--install-workflows` for explicit non-interactive
|
|
16
|
+
installation or `--no-workflow-prompt` to start the daemon without asking.
|
|
17
|
+
After installation, restart Codex and install SIMY from **Plugins → Personal**
|
|
18
|
+
before using `@simy` in a new task.
|
|
19
|
+
|
|
12
20
|
This is the recommended installation because the CLI can safely install patch
|
|
13
21
|
and minor releases, and the daemon restarts only when no Agentic Loop task is active. To run
|
|
14
22
|
the interactive chat after installation, use `simy`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@awak-app/simy-cli",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.3",
|
|
4
4
|
"description": "Local SIMY executor for non-coding Local Tasks and Agentic Loop runs.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
"test:e2e:console:fullstack": "node ./scripts/fullstack-cli-smoke.js",
|
|
39
39
|
"check": "npm run check:auto-update && node --check ./src/desktop-executor.js && node --check ./src/local-task.js && node --check ./scripts/desktop-executor-e2e.js && node --check ./scripts/pre-guardrail-e2e.js && node --check ./scripts/fixtures/fake-desktop-provider.js && npm run check:syntax && npm run check:real-provider-syntax && npm test && npm run check:package",
|
|
40
40
|
"check:auto-update": "node --check ./src/auto-update.js && node --check ./scripts/auto-update-e2e.js && npm run check:sqm",
|
|
41
|
-
"check:sqm": "node --check ./src/sqm/canonical.js && node --check ./src/sqm/validation.js && node --check ./src/sqm/repository.js && node --check ./src/sqm/checkers.js && node --check ./src/sqm/bundle-store.js && node --check ./src/sqm/proof.js && node --check ./src/sqm/evidence-client.js && node --check ./src/sqm/session.js && node --check ./src/sqm/index.js && node --check ./src/sqm/command.js && node --check ./src/sqm/agent-skill.js && node --check ./src/sqm/review-worker.js && node --check ./scripts/sqm-complete-loop-e2e.js && node --check ./scripts/sqm-local-review-e2e.js",
|
|
41
|
+
"check:sqm": "node --check ./src/sqm/canonical.js && node --check ./src/sqm/validation.js && node --check ./src/sqm/reliability.js && node --check ./src/sqm/repository.js && node --check ./src/sqm/checkers.js && node --check ./src/sqm/bundle-store.js && node --check ./src/sqm/proof.js && node --check ./src/sqm/evidence-client.js && node --check ./src/sqm/session.js && node --check ./src/sqm/index.js && node --check ./src/sqm/command.js && node --check ./src/sqm/agent-skill.js && node --check ./src/sqm/workflow-onboarding.js && node --check ./src/sqm/review-worker.js && node --check ./scripts/sqm-complete-loop-e2e.js && node --check ./scripts/sqm-local-review-e2e.js",
|
|
42
42
|
"check:real-provider-syntax": "node --check ./scripts/real-codex-probe.js && node --check ./scripts/real-claude-probe.js && node --check ./scripts/real-provider-console-fixture.js && node --check ./scripts/real-provider-console-e2e.js",
|
|
43
43
|
"check:syntax": "node --check ./src/index.js && node --check ./src/agent.js && node --check ./src/browser.js && node --check ./src/web-api.js && node --check ./src/durable-local-task-worker.js && node --check ./src/durable-local-task-steps-contract.js && node --check ./src/bounded-local-task-subtasks-contract.js && node --check ./src/bounded-local-task-subtask-pool.js && node --check ./src/local-task-artifact-contract.js && node --check ./src/execution-capability-contract.js && node --check ./src/shutdown.js && node --check ./src/run-registry.js && node --check ./src/direct-executor.js && node --check ./src/execution-guardrail.js && node --check ./src/backend-executable.js && node --check ./src/provider-stream.js && node --check ./src/console/index.js && node --check ./src/console/app.js && node --check ./src/console/commands.js && node --check ./src/local-attachments.js && node --check ./src/repository-inventory.js && node --check ./src/session-store.js && node --check ./src/web-origin.js && node --check ./src/workspace-context.js && node --check ./src/orchestrator/index.js && node --check ./src/orchestrator/shared.js && node --check ./src/orchestrator/recovery.js && node --check ./src/orchestrator/risk.js && node --check ./src/orchestrator/contract.js && node --check ./src/orchestrator/completion-contract.js && node --check ./src/orchestrator/instruction.js && node --check ./src/orchestrator/problem-solving.js && node --check ./src/orchestrator/budget.js && node --check ./src/orchestrator/execution-io.js && node --check ./src/orchestrator/presentation.js && node --check ./src/orchestrator/result.js && node --check ./src/orchestrator/retry.js && node --check ./src/orchestrator/evidence.js && node --check ./src/orchestrator/independent-audit.js && node --check ./src/orchestrator/audit.js && node --check ./src/orchestrator/loop.js && node --check ./src/runner.js && node --check ./scripts/fixtures/completion-contract-fixture.js && node --check ./scripts/fixtures/fake-coding-backend.js && node --check ./scripts/e2e-terminal-evidence.js && node --check ./scripts/audit-gate-e2e.js && node --check ./scripts/completion-contract-e2e.js && node --check ./scripts/retry-circuit-e2e.js && node --check ./scripts/follow-up-charter-e2e.js && node --check ./scripts/task-routing-e2e.js && node --check ./scripts/executor-version-preflight-e2e.js && node --check ./scripts/blocked-recovery-e2e.js && node --check ./scripts/process-localization-e2e.js && node --check ./scripts/hypothesis-retries-e2e.js && node --check ./scripts/budget-enforcement-e2e.js && node --check ./scripts/local-task-default-workspace-e2e.js && node --check ./scripts/local-task-idempotency-e2e.js && node --check ./scripts/console-e2e-fixture.js && node --check ./scripts/console-e2e.js && node --check ./scripts/console-recovery-e2e-fixture.js && node --check ./scripts/console-recovery-e2e.js && node --check ./scripts/real-provider-console-fixture.js && node --check ./scripts/real-provider-console-e2e.js && node --check ./scripts/fullstack-cli-smoke.js && node --check ./scripts/check-package-contents.js",
|
|
44
44
|
"check:package": "node ./scripts/check-package-contents.js"
|
package/src/cli-contract.js
CHANGED
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
} from "./durable-local-task-steps-contract.js";
|
|
14
14
|
import { buildExecutionCapabilityContract } from "./execution-capability-contract.js";
|
|
15
15
|
|
|
16
|
-
export const CLI_VERSION = "0.5.
|
|
16
|
+
export const CLI_VERSION = "0.5.3";
|
|
17
17
|
export const CLI_API_CONTRACT_VERSION = 10;
|
|
18
18
|
|
|
19
19
|
export function withCliContract(
|
package/src/index.js
CHANGED
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
createAgentShutdown,
|
|
20
20
|
installAgentShutdownSignalHandlers,
|
|
21
21
|
} from "./shutdown.js";
|
|
22
|
+
import { prepareDaemonWorkflows } from "./sqm/workflow-onboarding.js";
|
|
22
23
|
import { DEFAULT_WEB_ORIGIN, resolveWebOrigin } from "./web-origin.js";
|
|
23
24
|
|
|
24
25
|
const argv = process.argv.slice(2);
|
|
@@ -90,6 +91,8 @@ async function runCommand() {
|
|
|
90
91
|
simy --no-tui Start the foreground HTTP agent without the interactive console
|
|
91
92
|
simy sqm min-check Run the fast deterministic SQM diff preflight
|
|
92
93
|
simy sqm check Run full scenarios, write proof.json, and sign the run
|
|
94
|
+
simy sqm lint Validate module reliability before publication
|
|
95
|
+
simy sqm replay Replay modules against labelled historical snapshots
|
|
93
96
|
simy update --check Check the installed and latest stable CLI versions
|
|
94
97
|
simy update Install an eligible stable CLI update
|
|
95
98
|
|
|
@@ -99,6 +102,8 @@ Options:
|
|
|
99
102
|
--no-open Do not open the browser when authorization is required
|
|
100
103
|
--port <port> Bind a specific localhost port
|
|
101
104
|
--host <url> Connect to a SIMY Web origin (default: ${DEFAULT_WEB_ORIGIN})
|
|
105
|
+
--install-workflows Install or update SIMY Codex/Claude skills and PR hooks
|
|
106
|
+
--no-workflow-prompt Skip the daemon startup workflow-installation prompt
|
|
102
107
|
--no-auto-update Disable update checks and automatic installation
|
|
103
108
|
|
|
104
109
|
${autoUpdateHelpText()}
|
|
@@ -122,6 +127,14 @@ async function runLocalAgent() {
|
|
|
122
127
|
return;
|
|
123
128
|
}
|
|
124
129
|
|
|
130
|
+
try {
|
|
131
|
+
await prepareDaemonWorkflows({ argv, daemon });
|
|
132
|
+
} catch (error) {
|
|
133
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
134
|
+
process.exitCode = 2;
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
|
|
125
138
|
if (daemon && process.env.SIMY_DAEMON_CHILD !== "1") {
|
|
126
139
|
const child = spawn(process.execPath, [fileURLToPath(import.meta.url), ...process.argv.slice(2)], {
|
|
127
140
|
detached: true,
|
package/src/sqm/checkers.js
CHANGED
|
@@ -2,6 +2,8 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import { lstat, readFile } from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
|
|
5
|
+
import { lintKnowledgeModule } from "./reliability.js";
|
|
6
|
+
|
|
5
7
|
const MAX_FILE_BYTES = 1024 * 1024;
|
|
6
8
|
const MAX_FILES_PER_PATTERN = 5_000;
|
|
7
9
|
|
|
@@ -12,10 +14,25 @@ export async function executeKnowledgeBundle(bundle, repository) {
|
|
|
12
14
|
export async function evaluateKnowledgeBundle(bundle, repository) {
|
|
13
15
|
const findings = [];
|
|
14
16
|
const modules = [];
|
|
17
|
+
const matcherDiagnostics = [];
|
|
15
18
|
for (const module of bundle.modules) {
|
|
16
|
-
if (["candidate", "retired"].includes(module.lifecycle.status))
|
|
17
|
-
|
|
18
|
-
|
|
19
|
+
if (["candidate", "retired"].includes(module.lifecycle.status)) {
|
|
20
|
+
matcherDiagnostics.push(moduleDiagnostic(module, "inactive_lifecycle"));
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
if (!moduleApplies(module, repository.repository)) {
|
|
24
|
+
matcherDiagnostics.push(moduleDiagnostic(module, "repository_scope_mismatch"));
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
const matcherTrace = await matchModule(module, repository);
|
|
28
|
+
matcherDiagnostics.push(moduleDiagnostic(
|
|
29
|
+
module,
|
|
30
|
+
matcherTrace.matched ? "matched" : "matcher_not_satisfied",
|
|
31
|
+
matcherTrace.matchers,
|
|
32
|
+
));
|
|
33
|
+
if (!matcherTrace.matched) continue;
|
|
34
|
+
const reliability = lintKnowledgeModule(module);
|
|
35
|
+
const enforcement = reliability.eligible_for_enforcement ? "blocking" : "advisory";
|
|
19
36
|
const exceptions = activeExceptionPatterns(module);
|
|
20
37
|
const ruleResults = [];
|
|
21
38
|
for (const rule of module.detection.checks) {
|
|
@@ -32,6 +49,8 @@ export async function evaluateKnowledgeBundle(bundle, repository) {
|
|
|
32
49
|
module_version: module.version,
|
|
33
50
|
rule_id: rule.id,
|
|
34
51
|
lifecycle: module.lifecycle.status,
|
|
52
|
+
enforcement,
|
|
53
|
+
reliability_issues: reliability.issues.map((issue) => issue.code),
|
|
35
54
|
severity: module.lifecycle.severity,
|
|
36
55
|
message: rule.message,
|
|
37
56
|
invariant: module.invariant.statement,
|
|
@@ -49,15 +68,31 @@ export async function evaluateKnowledgeBundle(bundle, repository) {
|
|
|
49
68
|
stressor_ids: assurance.stressor_ids,
|
|
50
69
|
strength_ids: assurance.strength_ids,
|
|
51
70
|
scenario_ids: assurance.scenario_ids,
|
|
71
|
+
matcher_trace: matcherTrace.matchers,
|
|
52
72
|
});
|
|
53
73
|
}
|
|
54
74
|
}
|
|
55
|
-
modules.push(evaluateAssurance(module, ruleResults
|
|
75
|
+
modules.push(evaluateAssurance(module, ruleResults, {
|
|
76
|
+
enforcement,
|
|
77
|
+
reliability,
|
|
78
|
+
matcherTrace: matcherTrace.matchers,
|
|
79
|
+
}));
|
|
56
80
|
}
|
|
57
|
-
return { findings, modules };
|
|
81
|
+
return { findings, modules, matcherDiagnostics };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function moduleDiagnostic(module, reason, matchers = []) {
|
|
85
|
+
return {
|
|
86
|
+
module_id: module.id,
|
|
87
|
+
module_version: module.version,
|
|
88
|
+
lifecycle: module.lifecycle.status,
|
|
89
|
+
applicable: reason === "matched",
|
|
90
|
+
reason,
|
|
91
|
+
matchers,
|
|
92
|
+
};
|
|
58
93
|
}
|
|
59
94
|
|
|
60
|
-
function evaluateAssurance(module, ruleResults) {
|
|
95
|
+
function evaluateAssurance(module, ruleResults, { enforcement, reliability, matcherTrace }) {
|
|
61
96
|
const failed = new Set(ruleResults.filter((result) => result.status === "failed").map((result) => result.rule_id));
|
|
62
97
|
const assurance = module.assurance;
|
|
63
98
|
if (!assurance) {
|
|
@@ -66,6 +101,9 @@ function evaluateAssurance(module, ruleResults) {
|
|
|
66
101
|
module_version: module.version,
|
|
67
102
|
lifecycle: module.lifecycle.status,
|
|
68
103
|
severity: module.lifecycle.severity,
|
|
104
|
+
enforcement,
|
|
105
|
+
reliability,
|
|
106
|
+
matcher_trace: matcherTrace,
|
|
69
107
|
rule_results: ruleResults,
|
|
70
108
|
state_models: [],
|
|
71
109
|
stressors: [],
|
|
@@ -85,11 +123,14 @@ function evaluateAssurance(module, ruleResults) {
|
|
|
85
123
|
}));
|
|
86
124
|
const strengthStatus = new Map(strengths.map((strength) => [strength.id, strength.status]));
|
|
87
125
|
const scenarios = assurance.scenarios.map((scenario) => {
|
|
88
|
-
|
|
126
|
+
const checkFailed = scenario.check_refs.some((ref) => failed.has(ref));
|
|
127
|
+
const strengthFailed = scenario.expected_strength_refs.some((ref) => strengthStatus.get(ref) === "failed");
|
|
128
|
+
const activated = checkFailed || strengthFailed;
|
|
129
|
+
let status = "passed";
|
|
89
130
|
if (scenario.execution_requirement === "local") {
|
|
90
|
-
const checkFailed = scenario.check_refs.some((ref) => failed.has(ref));
|
|
91
|
-
const strengthFailed = scenario.expected_strength_refs.some((ref) => strengthStatus.get(ref) === "failed");
|
|
92
131
|
status = checkFailed || strengthFailed ? "failed" : "passed";
|
|
132
|
+
} else if (activated && enforcement === "blocking") {
|
|
133
|
+
status = scenario.execution_requirement;
|
|
93
134
|
}
|
|
94
135
|
return {
|
|
95
136
|
id: scenario.id,
|
|
@@ -99,6 +140,7 @@ function evaluateAssurance(module, ruleResults) {
|
|
|
99
140
|
stressor_refs: [...scenario.stressor_refs],
|
|
100
141
|
expected_strength_refs: [...scenario.expected_strength_refs],
|
|
101
142
|
check_refs: [...scenario.check_refs],
|
|
143
|
+
activated,
|
|
102
144
|
};
|
|
103
145
|
});
|
|
104
146
|
const scenarioStatus = new Map(scenarios.map((scenario) => [scenario.id, scenario.status]));
|
|
@@ -120,6 +162,9 @@ function evaluateAssurance(module, ruleResults) {
|
|
|
120
162
|
module_version: module.version,
|
|
121
163
|
lifecycle: module.lifecycle.status,
|
|
122
164
|
severity: module.lifecycle.severity,
|
|
165
|
+
enforcement,
|
|
166
|
+
reliability,
|
|
167
|
+
matcher_trace: matcherTrace,
|
|
123
168
|
rule_results: ruleResults,
|
|
124
169
|
state_models: stateModels,
|
|
125
170
|
stressors,
|
|
@@ -146,24 +191,39 @@ function moduleApplies(module, repository) {
|
|
|
146
191
|
}
|
|
147
192
|
|
|
148
193
|
async function matchModule(module, repository) {
|
|
149
|
-
|
|
150
|
-
|
|
194
|
+
const traces = [];
|
|
195
|
+
for (const [index, matcher] of module.detection.matchers.entries()) {
|
|
196
|
+
const result = await match(matcher, repository);
|
|
197
|
+
traces.push({
|
|
198
|
+
index,
|
|
199
|
+
type: matcher.type,
|
|
200
|
+
paths: [...matcher.paths],
|
|
201
|
+
matched: result.matched,
|
|
202
|
+
matched_files: result.matchedFiles,
|
|
203
|
+
});
|
|
151
204
|
}
|
|
152
|
-
return
|
|
205
|
+
return { matched: traces.every((trace) => trace.matched), matchers: traces };
|
|
153
206
|
}
|
|
154
207
|
|
|
155
208
|
async function match(matcher, repository) {
|
|
156
209
|
const changed = repository.diff.filter((file) => matchesAny(matcher.paths, file.path));
|
|
157
|
-
if (matcher.type === "changed_path") return changed.length > 0;
|
|
210
|
+
if (matcher.type === "changed_path") return matchResult(changed.length > 0, changed.map((file) => file.path));
|
|
158
211
|
const regex = safeRegex(matcher.pattern);
|
|
159
212
|
if (matcher.type === "diff_regex" || matcher.type === "reference") {
|
|
160
|
-
|
|
213
|
+
const matched = changed.filter((file) => file.lines.some((line) => sideMatches(matcher.side, line.side) && test(regex, line.text)));
|
|
214
|
+
return matchResult(matched.length > 0, matched.map((file) => file.path));
|
|
161
215
|
}
|
|
162
216
|
if (matcher.type === "content_regex") {
|
|
163
|
-
const files =
|
|
164
|
-
|
|
217
|
+
const files = changed.map((file) => file.path);
|
|
218
|
+
const matched = [];
|
|
219
|
+
for (const file of files) if (test(regex, await readText(repository.root, file))) matched.push(file);
|
|
220
|
+
return matchResult(matched.length > 0, matched);
|
|
165
221
|
}
|
|
166
|
-
return false;
|
|
222
|
+
return matchResult(false, []);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function matchResult(matched, matchedFiles) {
|
|
226
|
+
return { matched, matchedFiles: [...new Set(matchedFiles)].slice(0, 50) };
|
|
167
227
|
}
|
|
168
228
|
|
|
169
229
|
async function executeRule(rule, repository) {
|
package/src/sqm/command.js
CHANGED
|
@@ -5,7 +5,9 @@ import path from "node:path";
|
|
|
5
5
|
import { isSessionValid, readSession } from "../session-store.js";
|
|
6
6
|
import { resolveWebOrigin } from "../web-origin.js";
|
|
7
7
|
import { uploadRunEvidence } from "./evidence-client.js";
|
|
8
|
+
import { evaluateKnowledgeBundle } from "./checkers.js";
|
|
8
9
|
import { runSqmCheck, runSqmMinCheck } from "./index.js";
|
|
10
|
+
import { lintKnowledgeModule, summarizeReplay } from "./reliability.js";
|
|
9
11
|
import { ensureSqmSession } from "./session.js";
|
|
10
12
|
|
|
11
13
|
export async function runSqmCommand(argv, { cliVersion }) {
|
|
@@ -13,7 +15,9 @@ export async function runSqmCommand(argv, { cliVersion }) {
|
|
|
13
15
|
printHelp();
|
|
14
16
|
return 0;
|
|
15
17
|
}
|
|
16
|
-
if (
|
|
18
|
+
if (argv[0] === "lint") return await runLintCommand(argv);
|
|
19
|
+
if (argv[0] === "replay") return await runReplayCommand(argv);
|
|
20
|
+
if (!["min-check", "check"].includes(argv[0])) throw new Error("Usage: simy sqm <min-check|check|lint|replay> [options]");
|
|
17
21
|
const webOrigin = resolveWebOrigin(option(argv, "--host"));
|
|
18
22
|
let session = await readSession(webOrigin);
|
|
19
23
|
let validSession = isSessionValid(session, Date.now(), webOrigin) ? session : null;
|
|
@@ -61,14 +65,27 @@ export async function runSqmCommand(argv, { cliVersion }) {
|
|
|
61
65
|
const result = await runSqmCheck({ ...common, minCheck, deviceId, signingKeyPath: option(argv, "--signing-key") || undefined });
|
|
62
66
|
const proofPath = path.resolve(option(argv, "--proof") || "proof.json");
|
|
63
67
|
const evidencePath = path.resolve(option(argv, "--signed-evidence") || `${proofPath}.signed.json`);
|
|
64
|
-
|
|
68
|
+
const diagnosticsPath = path.resolve(option(argv, "--diagnostics") || `${proofPath}.diagnostics.json`);
|
|
69
|
+
await Promise.all([
|
|
70
|
+
atomicJson(proofPath, result.proof),
|
|
71
|
+
atomicJson(evidencePath, result.signed_evidence),
|
|
72
|
+
atomicJson(diagnosticsPath, {
|
|
73
|
+
policy_version: "sqm-reliability.v1",
|
|
74
|
+
matcher_diagnostics: result.matcher_diagnostics,
|
|
75
|
+
}),
|
|
76
|
+
]);
|
|
65
77
|
let upload = { uploaded: false, reason: "disabled" };
|
|
66
78
|
if (!argv.includes("--no-upload")) {
|
|
67
79
|
const evidenceEndpoint = process.env.SIMY_SQM_EVIDENCE_URL || (validSession?.api_base_url ? new URL("sqm/run-evidence", validSession.api_base_url).href : undefined);
|
|
68
80
|
upload = await uploadRunEvidence({ endpoint: evidenceEndpoint, token, proof: result.proof, signedEvidence: result.signed_evidence });
|
|
69
81
|
}
|
|
70
82
|
result.upload = upload;
|
|
71
|
-
result.artifacts = {
|
|
83
|
+
result.artifacts = {
|
|
84
|
+
min_check: minPath,
|
|
85
|
+
proof: proofPath,
|
|
86
|
+
signed_evidence: evidencePath,
|
|
87
|
+
diagnostics: diagnosticsPath,
|
|
88
|
+
};
|
|
72
89
|
if (argv.includes("--json")) console.log(JSON.stringify(result, null, 2));
|
|
73
90
|
else printFull(result);
|
|
74
91
|
return result.proof.outcome.status === "failed" ? 1 : 0;
|
|
@@ -78,12 +95,16 @@ function printHelp() {
|
|
|
78
95
|
console.log(`Usage:
|
|
79
96
|
simy sqm min-check [options]
|
|
80
97
|
simy sqm check [options]
|
|
98
|
+
simy sqm lint --bundle <path>
|
|
99
|
+
simy sqm replay --bundle <path> --dataset <path>
|
|
81
100
|
|
|
82
101
|
Two-stage SQM flow:
|
|
83
102
|
min-check Fast deterministic diff preflight and full-check execution plan.
|
|
84
103
|
check Full state/invariant/stress/strength/scenario verification. It
|
|
85
104
|
verifies the pinned min-check identity, writes proof.json, and
|
|
86
105
|
produces independent per-run signed evidence.
|
|
106
|
+
lint Report broad matchers and missing evaluation gates before publish.
|
|
107
|
+
replay Evaluate a bundle against labelled historical repository snapshots.
|
|
87
108
|
|
|
88
109
|
Options:
|
|
89
110
|
--base <branch> Compare against this base branch (default: dev)
|
|
@@ -95,12 +116,75 @@ Options:
|
|
|
95
116
|
--min-result <path> full check input (default: the last local min-check)
|
|
96
117
|
--proof <path> proof.json output (default: ./proof.json)
|
|
97
118
|
--signed-evidence <path> per-run signature output
|
|
119
|
+
--diagnostics <path> local matcher trace output
|
|
98
120
|
--no-upload keep proof local instead of uploading hashes/results
|
|
99
121
|
--json print machine-readable output
|
|
100
122
|
|
|
101
123
|
Neither phase executes Cloud-provided code or calls a local LLM.`);
|
|
102
124
|
}
|
|
103
125
|
|
|
126
|
+
async function runLintCommand(argv) {
|
|
127
|
+
const bundlePath = requiredOption(argv, "--bundle");
|
|
128
|
+
const input = await readJson(bundlePath);
|
|
129
|
+
const modules = Array.isArray(input.modules) ? input.modules : [input];
|
|
130
|
+
const report = {
|
|
131
|
+
policy_version: "sqm-reliability.v1",
|
|
132
|
+
modules: modules.map((module) => ({
|
|
133
|
+
module_id: module?.id ?? null,
|
|
134
|
+
lifecycle: module?.lifecycle?.status ?? null,
|
|
135
|
+
...lintKnowledgeModule(module),
|
|
136
|
+
})),
|
|
137
|
+
};
|
|
138
|
+
report.issue_count = report.modules.reduce((total, module) => total + module.issues.length, 0);
|
|
139
|
+
if (argv.includes("--json")) console.log(JSON.stringify(report, null, 2));
|
|
140
|
+
else {
|
|
141
|
+
console.log(`SQM lint: ${report.issue_count} issue(s) across ${report.modules.length} module(s)`);
|
|
142
|
+
for (const module of report.modules) {
|
|
143
|
+
for (const issue of module.issues) console.log(`- ${module.module_id}: ${issue.code} · ${issue.message}`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return report.modules.some((module) => module.lifecycle === "blocking" && !module.eligible_for_enforcement) ? 1 : 0;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function runReplayCommand(argv) {
|
|
150
|
+
const bundlePath = requiredOption(argv, "--bundle");
|
|
151
|
+
const datasetPath = requiredOption(argv, "--dataset");
|
|
152
|
+
const [bundle, dataset] = await Promise.all([readJson(bundlePath), readJson(datasetPath)]);
|
|
153
|
+
if (!Array.isArray(bundle?.modules)) throw new Error("Replay bundle must contain a modules array.");
|
|
154
|
+
if (!Array.isArray(dataset?.cases) || !dataset.cases.length) throw new Error("Replay dataset must contain at least one labelled case.");
|
|
155
|
+
const results = [];
|
|
156
|
+
for (const [index, item] of dataset.cases.entries()) {
|
|
157
|
+
if (!["trigger", "clean"].includes(item?.expected)) throw new Error(`Replay case ${index} expected must be trigger or clean.`);
|
|
158
|
+
const repository = item.repository;
|
|
159
|
+
if (!repository?.repository || !Array.isArray(repository.paths) || !Array.isArray(repository.diff)) {
|
|
160
|
+
throw new Error(`Replay case ${index} must contain a repository snapshot.`);
|
|
161
|
+
}
|
|
162
|
+
const evaluation = await evaluateKnowledgeBundle(bundle, { root: repository.root || process.cwd(), ...repository });
|
|
163
|
+
results.push({
|
|
164
|
+
id: item.id || `case-${index + 1}`,
|
|
165
|
+
expected: item.expected,
|
|
166
|
+
triggered: evaluation.findings.length > 0,
|
|
167
|
+
finding_count: evaluation.findings.length,
|
|
168
|
+
module_ids: [...new Set(evaluation.findings.map((finding) => finding.module_id))],
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
const metrics = summarizeReplay(results);
|
|
172
|
+
const report = { policy_version: "sqm-reliability.v1", dataset: dataset.name || path.basename(datasetPath), metrics, results };
|
|
173
|
+
if (argv.includes("--json")) console.log(JSON.stringify(report, null, 2));
|
|
174
|
+
else console.log(`SQM replay: ${metrics.sample_count} case(s) · precision ${(metrics.precision * 100).toFixed(1)}% · recall ${(metrics.recall * 100).toFixed(1)}% · ${metrics.false_positives} false positive(s)`);
|
|
175
|
+
return metrics.false_positives || metrics.false_negatives ? 1 : 0;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
async function readJson(location) {
|
|
179
|
+
return JSON.parse(await readFile(path.resolve(location), "utf8"));
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function requiredOption(argv, name) {
|
|
183
|
+
const value = option(argv, name);
|
|
184
|
+
if (!value) throw new Error(`${name} is required.`);
|
|
185
|
+
return value;
|
|
186
|
+
}
|
|
187
|
+
|
|
104
188
|
function explicitSqmEnvironment(argv) {
|
|
105
189
|
const organization = process.env.SIMY_SQM_ORGANIZATION_ID?.trim();
|
|
106
190
|
const account = process.env.SIMY_SQM_ACCOUNT_ID?.trim();
|
|
@@ -123,7 +207,7 @@ function printFull(result) {
|
|
|
123
207
|
console.log(`SQM full check: ${result.proof.outcome.status} · ${result.findings.length} finding(s)`);
|
|
124
208
|
console.log(`Proof ${result.proof.proof_id} (${result.proof.integrity.digest})`);
|
|
125
209
|
console.log(`Per-run signature: valid · ${result.signature_verification.key_id}`);
|
|
126
|
-
console.log(`Artifacts: ${result.artifacts.proof} · ${result.artifacts.signed_evidence}`);
|
|
210
|
+
console.log(`Artifacts: ${result.artifacts.proof} · ${result.artifacts.signed_evidence} · ${result.artifacts.diagnostics}`);
|
|
127
211
|
if (result.upload.uploaded) console.log(`Cloud evidence: stored · ${result.upload.evidence_id || result.signed_evidence.evidence_id}`);
|
|
128
212
|
else console.log(`Cloud evidence: local only (${result.upload.reason})`);
|
|
129
213
|
}
|
package/src/sqm/index.js
CHANGED
|
@@ -99,7 +99,7 @@ async function buildMinCheck({ repository, bundle, evaluation, organizationId, c
|
|
|
99
99
|
knowledge_bundle: bundleIdentity(bundle),
|
|
100
100
|
executed_at: new Date().toISOString(),
|
|
101
101
|
cli_version: cliVersion,
|
|
102
|
-
status: evaluation.findings.
|
|
102
|
+
status: evaluation.findings.some((finding) => finding.enforcement === "blocking") ? "preliminary_failed" : "passed",
|
|
103
103
|
preliminary_findings: evaluation.findings,
|
|
104
104
|
matched_modules: evaluation.modules.map((module) => ({ module_id: module.module_id, module_version: module.module_version, state_model_ids: module.state_models.map((model) => model.id), transition_ids: module.state_models.flatMap((model) => model.transitions.map((transition) => transition.id)), stressor_ids: module.stressors.map((stressor) => stressor.id), scenario_ids: module.scenarios.map((scenario) => scenario.id) })),
|
|
105
105
|
full_check_plan: plan,
|
|
@@ -119,6 +119,7 @@ async function buildFullCheck({ repository, bundle, minCheck, evaluation, organi
|
|
|
119
119
|
min_check: minCheck,
|
|
120
120
|
findings: evaluation.findings,
|
|
121
121
|
executions: evaluation.modules,
|
|
122
|
+
matcher_diagnostics: evaluation.matcherDiagnostics || [],
|
|
122
123
|
proof,
|
|
123
124
|
signed_evidence: signedEvidence,
|
|
124
125
|
signature_verification: verification,
|
package/src/sqm/proof.js
CHANGED
|
@@ -38,13 +38,15 @@ export function verifyMinCheck(value) {
|
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
export function buildProof({ organizationId, repository, bundle, minCheck, evaluation, findings, cliVersion, executedAt = new Date().toISOString() }) {
|
|
41
|
-
const
|
|
42
|
-
const
|
|
43
|
-
const
|
|
44
|
-
const
|
|
41
|
+
const enforcingModules = evaluation.modules.filter((module) => module.enforcement === "blocking");
|
|
42
|
+
const enforcingFindings = findings.filter((finding) => finding.enforcement === "blocking");
|
|
43
|
+
const statuses = enforcingModules.flatMap((module) => module.scenarios.map((scenario) => scenario.status));
|
|
44
|
+
const failedModules = enforcingModules.filter((module) => module.scenarios.some((scenario) => scenario.status === "failed"));
|
|
45
|
+
const hasFailure = enforcingFindings.length > 0 || failedModules.length > 0;
|
|
46
|
+
const advisoryFindingCount = findings.length - enforcingFindings.length;
|
|
45
47
|
const requiresHuman = statuses.includes("requires_human");
|
|
46
48
|
const requiresPostDeploy = statuses.includes("requires_post_deploy");
|
|
47
|
-
const status =
|
|
49
|
+
const status = hasFailure ? "failed" : requiresHuman ? "requires_human" : requiresPostDeploy ? "requires_post_deploy" : "passed";
|
|
48
50
|
const proof = {
|
|
49
51
|
schema_version: "1.0.0",
|
|
50
52
|
proof_id: `SQM-PROOF-${randomUUID()}`,
|
|
@@ -67,9 +69,9 @@ export function buildProof({ organizationId, repository, bundle, minCheck, evalu
|
|
|
67
69
|
outcome: {
|
|
68
70
|
status,
|
|
69
71
|
reasons: [
|
|
70
|
-
...(hasFailure
|
|
71
|
-
...(
|
|
72
|
-
...(evaluation.modules.length === 0 ? ["
|
|
72
|
+
...(hasFailure ? ["one_or_more_blocking_checks_failed"] : []),
|
|
73
|
+
...(advisoryFindingCount > 0 ? ["advisory_findings_observed"] : []),
|
|
74
|
+
...(evaluation.modules.length === 0 ? ["no_applicable_rules"] : []),
|
|
73
75
|
...(requiresHuman ? ["human_evidence_required"] : []),
|
|
74
76
|
...(requiresPostDeploy ? ["post_deploy_evidence_required"] : []),
|
|
75
77
|
],
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
const BROAD_PATHS = new Set(["**", "**/*", "src/**", "src/**/*"]);
|
|
2
|
+
const BROAD_EXTENSION_PATH = /^\*\*\/\*\.[a-z0-9{}*,]+$/i;
|
|
3
|
+
|
|
4
|
+
export const RELIABILITY_POLICY_VERSION = "sqm-reliability.v1";
|
|
5
|
+
|
|
6
|
+
export function lintKnowledgeModule(module) {
|
|
7
|
+
const issues = [];
|
|
8
|
+
const repositories = module?.scope?.repositories || [];
|
|
9
|
+
const matchers = module?.detection?.matchers || [];
|
|
10
|
+
const lifecycle = module?.lifecycle?.status;
|
|
11
|
+
const evaluation = module?.evaluation || {};
|
|
12
|
+
|
|
13
|
+
if (!repositories.length) {
|
|
14
|
+
issues.push(issue("repository_scope_required", "Module must declare at least one repository."));
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const broadMatchers = matchers.filter((matcher) =>
|
|
18
|
+
(matcher.paths || []).some(isBroadPath)
|
|
19
|
+
);
|
|
20
|
+
if (broadMatchers.length) {
|
|
21
|
+
issues.push(issue(
|
|
22
|
+
"broad_matcher",
|
|
23
|
+
"Matchers using repository-wide or top-level source globs are observation-only.",
|
|
24
|
+
{ matcher_types: [...new Set(broadMatchers.map((matcher) => matcher.type))] },
|
|
25
|
+
));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const hasDiffSignal = matchers.some((matcher) =>
|
|
29
|
+
["changed_path", "diff_regex", "reference"].includes(matcher.type) &&
|
|
30
|
+
(matcher.paths || []).some((path) => !isBroadPath(path))
|
|
31
|
+
);
|
|
32
|
+
if (matchers.some((matcher) => matcher.type === "content_regex") && !hasDiffSignal) {
|
|
33
|
+
issues.push(issue(
|
|
34
|
+
"content_matcher_requires_diff_scope",
|
|
35
|
+
"A content matcher must be combined with a narrow changed-path or diff signal.",
|
|
36
|
+
));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (["warn", "blocking"].includes(lifecycle)) {
|
|
40
|
+
const total = number(evaluation.true_positives) + number(evaluation.false_positives);
|
|
41
|
+
const precision = total > 0 ? number(evaluation.true_positives) / total : 0;
|
|
42
|
+
const minimumSamples = lifecycle === "blocking" ? 50 : 20;
|
|
43
|
+
const minimumPrecision = lifecycle === "blocking" ? 0.99 : 0.95;
|
|
44
|
+
if (evaluation.status !== "historical_replay" && evaluation.status !== "prospective") {
|
|
45
|
+
issues.push(issue("replay_required", "Warn and blocking modules require replay or prospective evaluation."));
|
|
46
|
+
}
|
|
47
|
+
if (total < minimumSamples) {
|
|
48
|
+
issues.push(issue("insufficient_positive_samples", `Lifecycle ${lifecycle} requires at least ${minimumSamples} adjudicated positive samples.`, { actual: total, required: minimumSamples }));
|
|
49
|
+
}
|
|
50
|
+
if (precision < minimumPrecision) {
|
|
51
|
+
issues.push(issue("precision_below_threshold", `Lifecycle ${lifecycle} requires precision of at least ${minimumPrecision}.`, { actual: precision, required: minimumPrecision }));
|
|
52
|
+
}
|
|
53
|
+
if (lifecycle === "blocking" && number(evaluation.false_negatives) > 0) {
|
|
54
|
+
issues.push(issue("blocking_false_negative", "Blocking modules cannot have a known false negative."));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return {
|
|
59
|
+
policy_version: RELIABILITY_POLICY_VERSION,
|
|
60
|
+
eligible_for_enforcement: lifecycle === "blocking" && issues.length === 0,
|
|
61
|
+
issues,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function summarizeReplay(results) {
|
|
66
|
+
const counts = { true_positives: 0, false_positives: 0, false_negatives: 0, true_negatives: 0 };
|
|
67
|
+
for (const result of results) {
|
|
68
|
+
if (result.expected === "trigger" && result.triggered) counts.true_positives += 1;
|
|
69
|
+
else if (result.expected === "clean" && result.triggered) counts.false_positives += 1;
|
|
70
|
+
else if (result.expected === "trigger") counts.false_negatives += 1;
|
|
71
|
+
else counts.true_negatives += 1;
|
|
72
|
+
}
|
|
73
|
+
const predictedPositive = counts.true_positives + counts.false_positives;
|
|
74
|
+
const actualPositive = counts.true_positives + counts.false_negatives;
|
|
75
|
+
return {
|
|
76
|
+
...counts,
|
|
77
|
+
sample_count: results.length,
|
|
78
|
+
precision: predictedPositive ? counts.true_positives / predictedPositive : 0,
|
|
79
|
+
recall: actualPositive ? counts.true_positives / actualPositive : 0,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function isBroadPath(path) {
|
|
84
|
+
const value = String(path || "").replace(/^\.\//, "");
|
|
85
|
+
return BROAD_PATHS.has(value) || BROAD_EXTENSION_PATH.test(value);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function number(value) {
|
|
89
|
+
return Number.isFinite(Number(value)) ? Number(value) : 0;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function issue(code, message, details = {}) {
|
|
93
|
+
return { code, message, ...details };
|
|
94
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { createInterface } from "node:readline/promises";
|
|
2
|
+
|
|
3
|
+
import { inspectSqmAgentSkill, installSqmAgentSkill } from "./agent-skill.js";
|
|
4
|
+
|
|
5
|
+
const INSTALL_FLAG = "--install-workflows";
|
|
6
|
+
const SKIP_PROMPT_FLAG = "--no-workflow-prompt";
|
|
7
|
+
|
|
8
|
+
export async function prepareDaemonWorkflows({
|
|
9
|
+
argv,
|
|
10
|
+
daemon,
|
|
11
|
+
daemonChild = process.env.SIMY_DAEMON_CHILD === "1",
|
|
12
|
+
stdinIsTTY = process.stdin.isTTY === true,
|
|
13
|
+
stdoutIsTTY = process.stdout.isTTY === true,
|
|
14
|
+
inspect = inspectSqmAgentSkill,
|
|
15
|
+
install = installSqmAgentSkill,
|
|
16
|
+
confirm = confirmWorkflowInstallation,
|
|
17
|
+
logger = console.log,
|
|
18
|
+
errorLogger = console.error,
|
|
19
|
+
} = {}) {
|
|
20
|
+
if (!daemon || daemonChild) return { outcome: "not_applicable" };
|
|
21
|
+
|
|
22
|
+
const installRequested = argv.includes(INSTALL_FLAG);
|
|
23
|
+
const promptDisabled = argv.includes(SKIP_PROMPT_FLAG);
|
|
24
|
+
if (installRequested && promptDisabled) {
|
|
25
|
+
throw new Error(`${INSTALL_FLAG} cannot be combined with ${SKIP_PROMPT_FLAG}`);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
let status;
|
|
29
|
+
try {
|
|
30
|
+
status = await inspect();
|
|
31
|
+
} catch (error) {
|
|
32
|
+
errorLogger(
|
|
33
|
+
`SIMY could not inspect the Codex/Claude workflows: ${message(error)}. The daemon will still start.`,
|
|
34
|
+
);
|
|
35
|
+
return { outcome: "inspection_failed", error };
|
|
36
|
+
}
|
|
37
|
+
if (status.installed) return { outcome: "already_installed", status };
|
|
38
|
+
if (promptDisabled) return { outcome: "skipped", status };
|
|
39
|
+
|
|
40
|
+
if (!installRequested && (!stdinIsTTY || !stdoutIsTTY)) {
|
|
41
|
+
logger(
|
|
42
|
+
"SIMY Codex/Claude workflows are missing or outdated. Re-run in a terminal with --install-workflows to install them.",
|
|
43
|
+
);
|
|
44
|
+
return { outcome: "confirmation_unavailable", status };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const accepted = installRequested || (await confirm({ status }));
|
|
48
|
+
if (!accepted) {
|
|
49
|
+
logger("SIMY workflow installation skipped. The daemon will still start.");
|
|
50
|
+
return { outcome: "declined", status };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
try {
|
|
54
|
+
const installed = await install();
|
|
55
|
+
logger(
|
|
56
|
+
"SIMY workflows installed. Restart Codex, then install SIMY from Plugins > Personal before using @simy.",
|
|
57
|
+
);
|
|
58
|
+
return { outcome: "installed", status: installed };
|
|
59
|
+
} catch (error) {
|
|
60
|
+
errorLogger(`SIMY workflow installation failed: ${message(error)}. The daemon will still start.`);
|
|
61
|
+
return { outcome: "installation_failed", error, status };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export async function confirmWorkflowInstallation({
|
|
66
|
+
input = process.stdin,
|
|
67
|
+
output = process.stdout,
|
|
68
|
+
} = {}) {
|
|
69
|
+
output.write(
|
|
70
|
+
[
|
|
71
|
+
"SIMY can install or update its Codex plugin marketplace, Codex/Claude SQM skills, and PR hooks.",
|
|
72
|
+
"Existing unrelated plugins and hooks will be preserved.",
|
|
73
|
+
"",
|
|
74
|
+
].join("\n"),
|
|
75
|
+
);
|
|
76
|
+
const prompt = createInterface({ input, output });
|
|
77
|
+
try {
|
|
78
|
+
const answer = await prompt.question("Install SIMY workflows now? [y/N] ");
|
|
79
|
+
return /^(y|yes)$/i.test(answer.trim());
|
|
80
|
+
} finally {
|
|
81
|
+
prompt.close();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function message(error) {
|
|
86
|
+
return error instanceof Error ? error.message : String(error);
|
|
87
|
+
}
|