@devflow-tools/cli 0.16.11 → 0.16.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/doctor.d.ts +5 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +278 -19
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/embeddings.d.ts +8 -0
- package/dist/commands/embeddings.d.ts.map +1 -0
- package/dist/commands/embeddings.js +26 -0
- package/dist/commands/embeddings.js.map +1 -0
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +41 -3
- package/dist/commands/server.js.map +1 -1
- package/dist/index.js +53 -2
- package/dist/index.js.map +1 -1
- package/dist/lib/with-server.d.ts.map +1 -1
- package/dist/lib/with-server.js +12 -6
- package/dist/lib/with-server.js.map +1 -1
- package/dist/plugin-files/.claude-plugin/plugin.json +1 -1
- package/dist/plugin-files/dist/command-registry.json +5 -5
- package/dist/plugin-files/dist/hooks/hook-daemon.js +32 -14
- package/dist/plugin-files/dist/hooks/memory-snapshot-cache.js +11 -0
- package/dist/plugin-files/dist/hooks/post-tool-use-failure.js +2 -2
- package/dist/plugin-files/dist/hooks/post-tool-use.js +6 -3
- package/dist/plugin-files/dist/hooks/pre-compact.js +3 -2
- package/dist/plugin-files/dist/hooks/pre-tool-use.js +3 -2
- package/dist/plugin-files/dist/hooks/server-bootstrap.js +22 -22
- package/dist/plugin-files/dist/hooks/session-end.js +4 -2
- package/dist/plugin-files/dist/hooks/stop.js +17 -2
- package/dist/plugin-files/dist/hooks/user-prompt-submit.js +15 -4
- package/dist/plugin-files/dist/skills/devflow:react/SKILL.md +8 -0
- package/dist/plugin-files/hooks/session-start +12 -15
- package/dist/plugin-files/package.json +7 -7
- package/dist/plugin-files/skills/react/SKILL.md +8 -0
- package/package.json +27 -27
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { DoctorReportV1 } from "@devflow-tools/sdk";
|
|
2
|
+
import type { EmbeddingModelStatus } from "@devflow-tools/memory-engine";
|
|
2
3
|
export type { DoctorMetric, DoctorReportV1, DoctorSection, DoctorSeverity, DoctorStatus, } from "@devflow-tools/sdk";
|
|
3
4
|
export type DoctorResult = DoctorReportV1;
|
|
4
5
|
export interface DoctorOptions {
|
|
@@ -31,6 +32,10 @@ export interface DoctorDependencies {
|
|
|
31
32
|
httpGet?: (url: string) => Promise<{
|
|
32
33
|
status: number;
|
|
33
34
|
}>;
|
|
35
|
+
inspectEmbeddingModel?: (options: {
|
|
36
|
+
homeDir?: string;
|
|
37
|
+
env?: NodeJS.ProcessEnv;
|
|
38
|
+
}) => Promise<EmbeddingModelStatus>;
|
|
34
39
|
}
|
|
35
40
|
export interface DoctorCommandIO {
|
|
36
41
|
stdout(output: string): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../src/commands/doctor.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../src/commands/doctor.ts"],"names":[],"mappings":"AA8BA,OAAO,KAAK,EAEV,cAAc,EAIf,MAAM,oBAAoB,CAAC;AAM5B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AAEzE,YAAY,EACV,YAAY,EACZ,cAAc,EACd,aAAa,EACb,cAAc,EACd,YAAY,GACb,MAAM,oBAAoB,CAAC;AAoC5B,MAAM,MAAM,YAAY,GAAG,cAAc,CAAC;AAE1C,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,UAAU,eAAe;IACvB,GAAG,CAAC,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACrC,GAAG,CAAC,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnC,cAAc,CAAC,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;CACzC;AAED,UAAU,cAAc;IACtB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAAC;IACtC,KAAK,IAAI,IAAI,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,cAAc,CAAC;IAChD,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvD,qBAAqB,CAAC,EAAE,CAAC,OAAO,EAAE;QAChC,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;KACzB,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAwmKD,wBAAsB,mBAAmB,CACvC,OAAO,GAAE,aAAkB,EAC3B,YAAY,GAAE,kBAAuB,GACpC,OAAO,CAAC,YAAY,CAAC,CA4BvB;AAED,wBAAgB,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;AACnD,wBAAgB,SAAS,CACvB,OAAO,EAAE,aAAa,EACtB,YAAY,CAAC,EAAE,kBAAkB,GAChC,OAAO,CAAC,YAAY,CAAC,CAAC;AAezB,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAoB7D;AAED,wBAAsB,gBAAgB,CACpC,OAAO,GAAE,aAAkB,EAC3B,EAAE,GAAE,eAGH,EACD,YAAY,GAAE,kBAAuB,GACpC,OAAO,CAAC,MAAM,CAAC,CAIjB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC;IACvE,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,MAAM,EAAE;QAAE,YAAY,EAAE,OAAO,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,SAAS,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/B,OAAO,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAChD,SAAS,EAAE;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAClC,MAAM,EAAE;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;IAC7B,SAAS,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC;IAC7D,UAAU,EAAE,MAAM,CAAC;CACpB;AAOD,wBAAsB,qBAAqB,CACzC,OAAO,SAAgB,GACtB,OAAO,CAAC,YAAY,CAAC,CA8DvB;AAED,wBAAsB,cAAc,IAAI,OAAO,CAAC;IAC9C,KAAK,EAAE,OAAO,CAAC;IACf,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAiBD;AAED,wBAAsB,iBAAiB,IAAI,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CAevE;AAED,wBAAsB,mBAAmB,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAChE,UAAU,EAAE,KAAK,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,EAAE,EAAE,MAAM,CAAC;QACX,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC,CAAC;CACJ,CAAC,CAsBD;AAED,wBAAgB,iBAAiB,IAAI;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,CASxE"}
|
package/dist/commands/doctor.js
CHANGED
|
@@ -6,11 +6,12 @@ import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep }
|
|
|
6
6
|
import { spawnSync } from "node:child_process";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
import chalk from "chalk";
|
|
9
|
-
import { DOCTOR_SCHEMA_VERSION, checkMcpHealth, loadPluginsWithDiagnostics, readMcpInventory, redactMcpInventory, selectHostAdapter, } from "@devflow-tools/sdk";
|
|
9
|
+
import { DOCTOR_SCHEMA_VERSION, checkMcpHealth, loadPluginsWithDiagnostics, readMcpInventory, redactMcpInventory, readRuntimeEndpointState, listRuntimeHttpCircuits, selectHostAdapter, } from "@devflow-tools/sdk";
|
|
10
10
|
import { resolveActivePlugin, scanClaudeSkillShadowing } from "../lib/plugin-setup.js";
|
|
11
11
|
import { serverStatus } from "./server.js";
|
|
12
12
|
import { inspectSkillHealth } from "./skill-health.js";
|
|
13
13
|
import { getGlobalDevFlowDbPath } from "@devflow-tools/database";
|
|
14
|
+
import { inspectEmbeddingModel } from "@devflow-tools/memory-engine";
|
|
14
15
|
function getProjectHash(rootPath) {
|
|
15
16
|
return createHash("sha256").update(rootPath).digest("hex").slice(0, 16);
|
|
16
17
|
}
|
|
@@ -131,6 +132,24 @@ async function checkRuntimeControlPlane(project, dependencies) {
|
|
|
131
132
|
severity: "warning",
|
|
132
133
|
});
|
|
133
134
|
}
|
|
135
|
+
const endpoint = readRuntimeEndpointState(home);
|
|
136
|
+
metrics.push({
|
|
137
|
+
key: 'runtime_endpoint_registry',
|
|
138
|
+
value: endpoint ?? 'not registered',
|
|
139
|
+
target: 'healthy while active, stopped or absent while idle',
|
|
140
|
+
ok: !endpoint || endpoint.status === 'healthy' || endpoint.status === 'stopped',
|
|
141
|
+
severity: endpoint?.status === 'degraded' ? 'error' : 'warning',
|
|
142
|
+
});
|
|
143
|
+
const now = (dependencies.now ?? Date.now)();
|
|
144
|
+
const openCircuits = listRuntimeHttpCircuits(home)
|
|
145
|
+
.filter(circuit => circuit.status === 'open' && (circuit.openUntil ?? 0) > now);
|
|
146
|
+
metrics.push({
|
|
147
|
+
key: 'runtime_http_circuits',
|
|
148
|
+
value: openCircuits,
|
|
149
|
+
target: [],
|
|
150
|
+
ok: openCircuits.length === 0,
|
|
151
|
+
severity: openCircuits.some(circuit => circuit.failures >= 2) ? 'error' : 'warning',
|
|
152
|
+
});
|
|
134
153
|
return { name: "Contract", status: sectionStatus(metrics), metrics };
|
|
135
154
|
}
|
|
136
155
|
function canonicalizeProject(project) {
|
|
@@ -574,7 +593,10 @@ function semanticToolOutcome(row) {
|
|
|
574
593
|
const data = doctorRecord(output?.data);
|
|
575
594
|
const status = typeof output?.status === "string" ? output.status : undefined;
|
|
576
595
|
const metricStatus = typeof row.metric_status === "string" ? row.metric_status : undefined;
|
|
577
|
-
const
|
|
596
|
+
const expectedPolicyBlock = Boolean(row.blocked)
|
|
597
|
+
&& typeof row.block_reason === "string"
|
|
598
|
+
&& row.block_reason.startsWith("DEVFLOW_");
|
|
599
|
+
const failed = !expectedPolicyBlock && (Boolean(row.error)
|
|
578
600
|
|| Boolean(row.failure_category)
|
|
579
601
|
|| output?.ok === false
|
|
580
602
|
|| output?.success === false
|
|
@@ -582,7 +604,7 @@ function semanticToolOutcome(row) {
|
|
|
582
604
|
|| Boolean(output?.error)
|
|
583
605
|
|| status === "failed"
|
|
584
606
|
|| status === "rejected"
|
|
585
|
-
|| metricStatus === "error";
|
|
607
|
+
|| metricStatus === "error");
|
|
586
608
|
const degraded = !failed && (output?.degraded === true
|
|
587
609
|
|| Boolean(output?.degradation)
|
|
588
610
|
|| data?.degraded === true
|
|
@@ -611,7 +633,7 @@ function rate(count, total) {
|
|
|
611
633
|
function coverage(total, vectors) {
|
|
612
634
|
return total === 0 ? 1 : vectors / total;
|
|
613
635
|
}
|
|
614
|
-
function checkVectors(project, dependencies) {
|
|
636
|
+
async function checkVectors(project, dependencies) {
|
|
615
637
|
const metrics = [];
|
|
616
638
|
let codegraphNodes = null;
|
|
617
639
|
let symbolVectors = null;
|
|
@@ -690,8 +712,21 @@ function checkVectors(project, dependencies) {
|
|
|
690
712
|
metrics.push({ key: "knowledge_vector_schema", value: missing, target: [], ok: false, severity: "warning" });
|
|
691
713
|
}
|
|
692
714
|
else {
|
|
693
|
-
const
|
|
694
|
-
const
|
|
715
|
+
const docColumns = doctorTableColumns(db, "knowledge_docs");
|
|
716
|
+
const activeClause = docColumns.has("status") ? "AND d.status = 'active'" : "";
|
|
717
|
+
const chunks = doctorCount(db, `
|
|
718
|
+
SELECT COUNT(*) count
|
|
719
|
+
FROM knowledge_chunks c
|
|
720
|
+
JOIN knowledge_docs d ON d.id = c.doc_id
|
|
721
|
+
WHERE 1 = 1 ${activeClause}
|
|
722
|
+
`);
|
|
723
|
+
const vectors = doctorCount(db, `
|
|
724
|
+
SELECT COUNT(*) count
|
|
725
|
+
FROM knowledge_vec_rowids v
|
|
726
|
+
JOIN knowledge_chunks c ON c.id = v.id
|
|
727
|
+
JOIN knowledge_docs d ON d.id = c.doc_id
|
|
728
|
+
WHERE 1 = 1 ${activeClause}
|
|
729
|
+
`);
|
|
695
730
|
const value = coverage(chunks, vectors);
|
|
696
731
|
const rows = db.prepare(`
|
|
697
732
|
SELECT d.source source, COUNT(c.id) chunks,
|
|
@@ -699,6 +734,7 @@ function checkVectors(project, dependencies) {
|
|
|
699
734
|
FROM knowledge_docs d
|
|
700
735
|
LEFT JOIN knowledge_chunks c ON c.doc_id = d.id
|
|
701
736
|
LEFT JOIN knowledge_vec_rowids v ON v.id = c.id
|
|
737
|
+
WHERE 1 = 1 ${activeClause}
|
|
702
738
|
GROUP BY d.source ORDER BY d.source
|
|
703
739
|
`).all();
|
|
704
740
|
const bySource = rows.map(row => {
|
|
@@ -737,11 +773,18 @@ function checkVectors(project, dependencies) {
|
|
|
737
773
|
metrics.push({ key: "observation_vector_schema", value: "observations/embeddings missing", target: "Batch 2 vector schema", ok: false, severity: "warning" });
|
|
738
774
|
}
|
|
739
775
|
else {
|
|
740
|
-
const
|
|
776
|
+
const observationColumns = doctorTableColumns(db, "observations");
|
|
777
|
+
const hasEligibilityColumns = observationColumns.has("quality_status")
|
|
778
|
+
&& observationColumns.has("resolution_state");
|
|
779
|
+
const eligibilityClause = hasEligibilityColumns
|
|
780
|
+
? "WHERE quality_status = 'accepted' AND resolution_state <> 'superseded'"
|
|
781
|
+
: "";
|
|
782
|
+
const observations = doctorCount(db, `SELECT COUNT(*) count FROM observations ${eligibilityClause}`);
|
|
741
783
|
const vectors = doctorCount(db, `
|
|
742
784
|
SELECT COUNT(DISTINCT e.source_id) count
|
|
743
785
|
FROM embeddings e JOIN observations o ON o.id = e.source_id
|
|
744
786
|
WHERE e.source_type = 'observation'
|
|
787
|
+
${hasEligibilityColumns ? "AND o.quality_status = 'accepted' AND o.resolution_state <> 'superseded'" : ""}
|
|
745
788
|
`);
|
|
746
789
|
const missing = Math.max(0, observations - vectors);
|
|
747
790
|
metrics.push({ key: "observations", value: observations, ok: true, severity: "info" });
|
|
@@ -771,7 +814,56 @@ function checkVectors(project, dependencies) {
|
|
|
771
814
|
metrics.push({ key: "embedder_error_log_read", value: error.message, target: "readable error log", ok: false, severity: "error" });
|
|
772
815
|
}
|
|
773
816
|
}
|
|
774
|
-
metrics.push({
|
|
817
|
+
metrics.push({
|
|
818
|
+
key: "embedder_errors_24h",
|
|
819
|
+
value: recentErrors,
|
|
820
|
+
target: "trend; current readiness is gated by model state and integrity",
|
|
821
|
+
ok: true,
|
|
822
|
+
severity: "info",
|
|
823
|
+
});
|
|
824
|
+
try {
|
|
825
|
+
const model = await (dependencies.inspectEmbeddingModel ?? inspectEmbeddingModel)({
|
|
826
|
+
homeDir: dependencies.homePath,
|
|
827
|
+
env: dependencies.env ?? process.env,
|
|
828
|
+
});
|
|
829
|
+
metrics.push({
|
|
830
|
+
key: "embedding_model_state",
|
|
831
|
+
value: {
|
|
832
|
+
state: model.state,
|
|
833
|
+
modelId: model.modelId,
|
|
834
|
+
revision: model.revision,
|
|
835
|
+
},
|
|
836
|
+
target: { state: "ready" },
|
|
837
|
+
ok: model.ready,
|
|
838
|
+
severity: "warning",
|
|
839
|
+
});
|
|
840
|
+
metrics.push({
|
|
841
|
+
key: "embedding_model_cache_root",
|
|
842
|
+
value: model.cacheRoot,
|
|
843
|
+
target: "stable shared DevFlow cache",
|
|
844
|
+
ok: true,
|
|
845
|
+
severity: "info",
|
|
846
|
+
});
|
|
847
|
+
metrics.push({
|
|
848
|
+
key: "embedding_model_integrity",
|
|
849
|
+
value: {
|
|
850
|
+
missingFiles: model.missingFiles,
|
|
851
|
+
corruptFiles: model.corruptFiles,
|
|
852
|
+
},
|
|
853
|
+
target: { missingFiles: [], corruptFiles: [] },
|
|
854
|
+
ok: model.missingFiles.length === 0 && model.corruptFiles.length === 0,
|
|
855
|
+
severity: "warning",
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
catch (error) {
|
|
859
|
+
metrics.push({
|
|
860
|
+
key: "embedding_model_inspection",
|
|
861
|
+
value: error.message,
|
|
862
|
+
target: "readable verified shared model cache",
|
|
863
|
+
ok: false,
|
|
864
|
+
severity: "warning",
|
|
865
|
+
});
|
|
866
|
+
}
|
|
775
867
|
return { name: "Vectors", status: sectionStatus(metrics), metrics };
|
|
776
868
|
}
|
|
777
869
|
function checkTelemetry(project, strict, dependencies) {
|
|
@@ -1054,6 +1146,9 @@ function checkTelemetry(project, strict, dependencies) {
|
|
|
1054
1146
|
toolEventColumns.has("error") ? "event.error" : "NULL error",
|
|
1055
1147
|
toolEventColumns.has("failure_category") ? "event.failure_category" : "NULL failure_category",
|
|
1056
1148
|
toolEventColumns.has("output") ? "event.output" : "NULL output",
|
|
1149
|
+
toolEventColumns.has("blocked") ? "event.blocked" : "0 blocked",
|
|
1150
|
+
toolEventColumns.has("block_reason") ? "event.block_reason" : "NULL block_reason",
|
|
1151
|
+
canProjectFilterEvents ? "event_session.status session_status" : "NULL session_status",
|
|
1057
1152
|
canJoinMetrics ? "metric.status metric_status" : "NULL metric_status",
|
|
1058
1153
|
].join(", ");
|
|
1059
1154
|
const eventRows = db.prepare(`
|
|
@@ -1066,11 +1161,20 @@ function checkTelemetry(project, strict, dependencies) {
|
|
|
1066
1161
|
`).all(...(canProjectFilterEvents ? [cutoff, project] : [cutoff]));
|
|
1067
1162
|
const semantic = eventRows.map(semanticToolOutcome);
|
|
1068
1163
|
const semanticFailures = semantic.filter(outcome => outcome.failed).length;
|
|
1164
|
+
const unresolvedSemanticFailures = semantic.filter((outcome, index) => outcome.failed
|
|
1165
|
+
&& eventRows[index]?.session_status !== "completed").length;
|
|
1069
1166
|
const semanticDegradations = semantic.filter(outcome => outcome.degraded).length;
|
|
1070
1167
|
metrics.push({ key: "semantic_tool_calls_24h", value: eventRows.length, ok: true, severity: "info" });
|
|
1071
|
-
metrics.push({ key: "semantic_tool_failures_24h", value: semanticFailures, target:
|
|
1168
|
+
metrics.push({ key: "semantic_tool_failures_24h", value: semanticFailures, target: "trend", ok: true, severity: "info" });
|
|
1169
|
+
metrics.push({
|
|
1170
|
+
key: "unresolved_semantic_tool_failures_24h",
|
|
1171
|
+
value: unresolvedSemanticFailures,
|
|
1172
|
+
target: 0,
|
|
1173
|
+
ok: unresolvedSemanticFailures === 0,
|
|
1174
|
+
severity: strictSeverity,
|
|
1175
|
+
});
|
|
1072
1176
|
metrics.push({ key: "semantic_tool_degradations_24h", value: semanticDegradations, target: 0, ok: semanticDegradations === 0, severity: strictSeverity });
|
|
1073
|
-
metrics.push({ key: "semantic_tool_failure_rate_24h", value: rate(semanticFailures, eventRows.length), target:
|
|
1177
|
+
metrics.push({ key: "semantic_tool_failure_rate_24h", value: rate(semanticFailures, eventRows.length), target: "trend", ok: true, severity: "info" });
|
|
1074
1178
|
metrics.push({ key: "semantic_tool_degradation_rate_24h", value: rate(semanticDegradations, eventRows.length), target: 0, ok: semanticDegradations === 0, severity: strictSeverity });
|
|
1075
1179
|
metrics.push({
|
|
1076
1180
|
key: "semantic_tool_project_scope",
|
|
@@ -1086,8 +1190,11 @@ function checkTelemetry(project, strict, dependencies) {
|
|
|
1086
1190
|
continue;
|
|
1087
1191
|
distribution[row.failure_category] = (distribution[row.failure_category] ?? 0) + 1;
|
|
1088
1192
|
}
|
|
1089
|
-
const
|
|
1090
|
-
|
|
1193
|
+
const unresolvedRows = eventRows.filter((row, index) => semantic[index]?.failed
|
|
1194
|
+
&& row.session_status !== "completed");
|
|
1195
|
+
const classifiedEvents = unresolvedRows.filter(row => typeof row.failure_category === "string"
|
|
1196
|
+
&& row.failure_category.trim().length > 0).length;
|
|
1197
|
+
const classificationCoverage = unresolvedRows.length === 0 ? 1 : classifiedEvents / unresolvedRows.length;
|
|
1091
1198
|
metrics.push({
|
|
1092
1199
|
key: "failure_categories_24h",
|
|
1093
1200
|
value: distribution,
|
|
@@ -1454,6 +1561,75 @@ function checkBenchmark(dependencies) {
|
|
|
1454
1561
|
}
|
|
1455
1562
|
return { name: "Benchmark", status: sectionStatus(metrics), metrics };
|
|
1456
1563
|
}
|
|
1564
|
+
function checkRetrievalQuality(project, dependencies) {
|
|
1565
|
+
const metrics = [];
|
|
1566
|
+
const home = dependencies.homePath ?? homedir();
|
|
1567
|
+
const path = getGlobalDevFlowDbPath(home);
|
|
1568
|
+
if (!existsSync(path)) {
|
|
1569
|
+
metrics.push({
|
|
1570
|
+
key: 'retrieval_quality_report',
|
|
1571
|
+
value: 'missing global database',
|
|
1572
|
+
target: 'passing retrieval-quality report',
|
|
1573
|
+
ok: false,
|
|
1574
|
+
severity: 'warning',
|
|
1575
|
+
});
|
|
1576
|
+
return { name: 'Retrieval quality', status: sectionStatus(metrics), metrics };
|
|
1577
|
+
}
|
|
1578
|
+
let db;
|
|
1579
|
+
try {
|
|
1580
|
+
db = doctorOpenDatabase(path, dependencies);
|
|
1581
|
+
if (!doctorTableExists(db, 'benchmark_reports'))
|
|
1582
|
+
throw new Error('benchmark_reports table missing');
|
|
1583
|
+
const row = db.prepare(`
|
|
1584
|
+
SELECT report_json, created_at
|
|
1585
|
+
FROM benchmark_reports
|
|
1586
|
+
WHERE suite_id = 'retrieval-quality' AND project_root = ?
|
|
1587
|
+
ORDER BY created_at DESC
|
|
1588
|
+
LIMIT 1
|
|
1589
|
+
`).get(project);
|
|
1590
|
+
if (!row?.report_json) {
|
|
1591
|
+
metrics.push({
|
|
1592
|
+
key: 'retrieval_quality_report',
|
|
1593
|
+
value: 'no project report',
|
|
1594
|
+
target: 'run retrieval-quality benchmark for this project',
|
|
1595
|
+
ok: false,
|
|
1596
|
+
severity: 'warning',
|
|
1597
|
+
});
|
|
1598
|
+
return { name: 'Retrieval quality', status: sectionStatus(metrics), metrics };
|
|
1599
|
+
}
|
|
1600
|
+
const report = JSON.parse(row.report_json);
|
|
1601
|
+
const ageMs = (dependencies.now ?? Date.now)() - Number(row.created_at ?? 0);
|
|
1602
|
+
metrics.push({
|
|
1603
|
+
key: 'retrieval_quality_report',
|
|
1604
|
+
value: { status: report.qualityStatus, ageMs, diagnostics: report.diagnostics ?? [] },
|
|
1605
|
+
target: { status: 'passed', maxAgeMs: BENCHMARK_MAX_AGE_MS },
|
|
1606
|
+
ok: report.qualityStatus === 'passed' && ageMs <= BENCHMARK_MAX_AGE_MS,
|
|
1607
|
+
severity: report.qualityStatus === 'failed' ? 'error' : 'warning',
|
|
1608
|
+
});
|
|
1609
|
+
for (const [key, passed] of Object.entries(report.gates ?? {})) {
|
|
1610
|
+
metrics.push({
|
|
1611
|
+
key: `retrieval_${key}`,
|
|
1612
|
+
value: report.metrics?.[key],
|
|
1613
|
+
target: report.thresholds?.[key],
|
|
1614
|
+
ok: passed,
|
|
1615
|
+
severity: 'error',
|
|
1616
|
+
});
|
|
1617
|
+
}
|
|
1618
|
+
}
|
|
1619
|
+
catch (error) {
|
|
1620
|
+
metrics.push({
|
|
1621
|
+
key: 'retrieval_quality_read',
|
|
1622
|
+
value: error.message,
|
|
1623
|
+
target: 'readable retrieval quality report',
|
|
1624
|
+
ok: false,
|
|
1625
|
+
severity: 'error',
|
|
1626
|
+
});
|
|
1627
|
+
}
|
|
1628
|
+
finally {
|
|
1629
|
+
db?.close();
|
|
1630
|
+
}
|
|
1631
|
+
return { name: 'Retrieval quality', status: sectionStatus(metrics), metrics };
|
|
1632
|
+
}
|
|
1457
1633
|
function hookDebugFootprint(root) {
|
|
1458
1634
|
if (!existsSync(root))
|
|
1459
1635
|
return { bytes: 0, files: 0 };
|
|
@@ -4454,6 +4630,50 @@ function checkMemoryQuality(project, dependencies) {
|
|
|
4454
4630
|
ok: true,
|
|
4455
4631
|
severity: "info",
|
|
4456
4632
|
});
|
|
4633
|
+
if (doctorTableExists(db, 'retrieval_access_log')) {
|
|
4634
|
+
const attribution = db.prepare(`
|
|
4635
|
+
SELECT access_type, COUNT(*) count
|
|
4636
|
+
FROM retrieval_access_log
|
|
4637
|
+
GROUP BY access_type
|
|
4638
|
+
`).all();
|
|
4639
|
+
metrics.push({
|
|
4640
|
+
key: 'memory_retrieval_attribution',
|
|
4641
|
+
value: Object.fromEntries(attribution.map(row => [row.access_type, Number(row.count)])),
|
|
4642
|
+
target: 'selected/exposed are distinct from adopted',
|
|
4643
|
+
ok: true,
|
|
4644
|
+
severity: 'info',
|
|
4645
|
+
});
|
|
4646
|
+
}
|
|
4647
|
+
if (doctorTableExists(db, 'observations')) {
|
|
4648
|
+
const observationColumns = doctorTableColumns(db, 'observations');
|
|
4649
|
+
if (!observationColumns.has('evidence_status')) {
|
|
4650
|
+
metrics.push({
|
|
4651
|
+
key: 'explicit_memory_evidence_schema',
|
|
4652
|
+
value: 'evidence_status missing',
|
|
4653
|
+
target: 'memory schema v8+',
|
|
4654
|
+
ok: false,
|
|
4655
|
+
severity: 'error',
|
|
4656
|
+
});
|
|
4657
|
+
}
|
|
4658
|
+
else {
|
|
4659
|
+
const evidence = db.prepare(`
|
|
4660
|
+
SELECT
|
|
4661
|
+
COUNT(*) total,
|
|
4662
|
+
SUM(CASE WHEN evidence_status = 'verified' THEN 1 ELSE 0 END) verified
|
|
4663
|
+
FROM observations
|
|
4664
|
+
WHERE source = 'explicit_intent' AND quality_status = 'accepted'
|
|
4665
|
+
`).get();
|
|
4666
|
+
const total = Number(evidence?.total ?? 0);
|
|
4667
|
+
const verified = Number(evidence?.verified ?? 0);
|
|
4668
|
+
metrics.push({
|
|
4669
|
+
key: 'explicit_memory_evidence_coverage',
|
|
4670
|
+
value: total > 0 ? verified / total : 1,
|
|
4671
|
+
target: 1,
|
|
4672
|
+
ok: total === 0 || verified === total,
|
|
4673
|
+
severity: 'warning',
|
|
4674
|
+
});
|
|
4675
|
+
}
|
|
4676
|
+
}
|
|
4457
4677
|
metrics.push({
|
|
4458
4678
|
key: "memory_access_distribution",
|
|
4459
4679
|
value: {
|
|
@@ -4487,13 +4707,41 @@ function checkKnowledgeQuality(project, dependencies) {
|
|
|
4487
4707
|
metrics.push({ key: "knowledge_schema", value: "knowledge_chunks missing", target: "Batch 3 knowledge schema", ok: false, severity: "warning" });
|
|
4488
4708
|
}
|
|
4489
4709
|
else {
|
|
4490
|
-
const
|
|
4710
|
+
const chunkColumns = doctorTableColumns(db, 'knowledge_chunks');
|
|
4711
|
+
const tokenExpression = chunkColumns.has('token_count')
|
|
4712
|
+
? 'COALESCE(token_count, CAST((length(content) + 3) / 4 AS INTEGER))'
|
|
4713
|
+
: 'CAST((length(content) + 3) / 4 AS INTEGER)';
|
|
4714
|
+
const lengths = db.prepare(`SELECT ${tokenExpression} length FROM knowledge_chunks ORDER BY ${tokenExpression}`).all().map(row => Number(row.length));
|
|
4491
4715
|
const p95Index = Math.max(0, Math.ceil(lengths.length * 0.95) - 1);
|
|
4492
4716
|
const p95 = lengths.length === 0 ? 0 : lengths[p95Index];
|
|
4493
|
-
metrics.push({ key: "knowledge_chunk_length_p95", value: p95, target: "<= 3000", ok: p95 <= 3000, severity: "
|
|
4717
|
+
metrics.push({ key: "knowledge_chunk_length_p95", value: p95, target: "<= 3000 tokens", ok: p95 <= 3000, severity: "error" });
|
|
4494
4718
|
const fts = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'knowledge_fts'").get();
|
|
4495
4719
|
const trigram = /tokenize\s*=\s*['\"]?trigram/i.test(fts?.sql ?? "");
|
|
4496
4720
|
metrics.push({ key: "knowledge_fts_tokenizer", value: trigram ? "trigram" : "missing/non-trigram", target: "trigram", ok: trigram, severity: "warning" });
|
|
4721
|
+
if (doctorTableExists(db, 'knowledge_source_readiness')) {
|
|
4722
|
+
const readiness = db.prepare(`
|
|
4723
|
+
SELECT source, expected_version, documents, chunks, vectors, coverage, status, failure_category
|
|
4724
|
+
FROM knowledge_source_readiness ORDER BY source
|
|
4725
|
+
`).all();
|
|
4726
|
+
const active = readiness.filter(row => Number(row.documents ?? 0) > 0 || Number(row.chunks ?? 0) > 0);
|
|
4727
|
+
const minimumCoverage = active.length > 0
|
|
4728
|
+
? Math.min(...active.map(row => Number(row.coverage ?? 0)))
|
|
4729
|
+
: 0;
|
|
4730
|
+
metrics.push({
|
|
4731
|
+
key: 'knowledge_source_readiness',
|
|
4732
|
+
value: readiness,
|
|
4733
|
+
target: 'all active sources current with >=95% vector coverage',
|
|
4734
|
+
ok: active.length > 0 && active.every(row => row.status === 'ready' && Number(row.coverage ?? 0) >= 0.95),
|
|
4735
|
+
severity: 'error',
|
|
4736
|
+
});
|
|
4737
|
+
metrics.push({
|
|
4738
|
+
key: 'knowledge_vector_coverage',
|
|
4739
|
+
value: minimumCoverage,
|
|
4740
|
+
target: '>= 0.95',
|
|
4741
|
+
ok: active.length > 0 && minimumCoverage >= 0.95,
|
|
4742
|
+
severity: 'error',
|
|
4743
|
+
});
|
|
4744
|
+
}
|
|
4497
4745
|
}
|
|
4498
4746
|
}
|
|
4499
4747
|
catch (error) {
|
|
@@ -4545,8 +4793,19 @@ async function checkUiReachability(dependencies) {
|
|
|
4545
4793
|
const { manifest, metrics } = acceptanceManifestMetrics(dependencies);
|
|
4546
4794
|
const routes = manifest?.ui?.routes ?? [];
|
|
4547
4795
|
metrics.push({ key: "dashboard_routes", value: routes.length, target: ">= 13", ok: manifest?.ui?.status === "green" && routes.length >= 13, severity: "warning" });
|
|
4796
|
+
const endpoint = readRuntimeEndpointState(dependencies.homePath ?? homedir());
|
|
4797
|
+
if ((!endpoint || endpoint.status === "stopped") && !dependencies.httpGet) {
|
|
4798
|
+
metrics.push({
|
|
4799
|
+
key: "dashboard_server_health",
|
|
4800
|
+
value: "inactive",
|
|
4801
|
+
target: "HTTP 200 when the runtime endpoint is active",
|
|
4802
|
+
ok: true,
|
|
4803
|
+
severity: "info",
|
|
4804
|
+
});
|
|
4805
|
+
return { name: "UI reachability", status: sectionStatus(metrics), metrics };
|
|
4806
|
+
}
|
|
4548
4807
|
const port = dependencies.env?.PORT ?? process.env.PORT ?? "13337";
|
|
4549
|
-
const healthUrl = `http://127.0.0.1:${port}/api/health`;
|
|
4808
|
+
const healthUrl = `${endpoint?.url ?? `http://127.0.0.1:${port}`}/api/health`;
|
|
4550
4809
|
try {
|
|
4551
4810
|
const response = await (dependencies.httpGet ?? defaultHttpGet)(healthUrl);
|
|
4552
4811
|
metrics.push({ key: "dashboard_server_health", value: { url: healthUrl, status: response.status }, target: "HTTP 200", ok: response.status === 200, severity: "warning" });
|
|
@@ -4567,7 +4826,7 @@ export async function runStructuredDoctor(options = {}, dependencies = {}) {
|
|
|
4567
4826
|
checkIsolation(requestedProject, dependencies),
|
|
4568
4827
|
checkDistill(canonical.path, dependencies),
|
|
4569
4828
|
checkLegacyPaths(canonical.path, dependencies),
|
|
4570
|
-
checkVectors(canonical.path, dependencies),
|
|
4829
|
+
await checkVectors(canonical.path, dependencies),
|
|
4571
4830
|
checkTelemetry(canonical.path, Boolean(options.strict), dependencies),
|
|
4572
4831
|
checkInjection(canonical.path, dependencies),
|
|
4573
4832
|
checkMemoryQuality(canonical.path, dependencies),
|
|
@@ -4575,14 +4834,14 @@ export async function runStructuredDoctor(options = {}, dependencies = {}) {
|
|
|
4575
4834
|
contract,
|
|
4576
4835
|
checkDeadcode(dependencies),
|
|
4577
4836
|
checkBenchmark(dependencies),
|
|
4837
|
+
checkRetrievalQuality(canonical.path, dependencies),
|
|
4578
4838
|
checkDataFace(dependencies),
|
|
4579
4839
|
checkRuntimeSecurity(dependencies),
|
|
4580
4840
|
await checkUiReachability(dependencies),
|
|
4581
4841
|
checkHookPathHealth(dependencies),
|
|
4582
4842
|
];
|
|
4583
|
-
const exitCode =
|
|
4584
|
-
? 1
|
|
4585
|
-
: 0;
|
|
4843
|
+
const exitCode = sections.some(section => section.status === 'red'
|
|
4844
|
+
|| (options.strict && section.status === 'yellow')) ? 1 : 0;
|
|
4586
4845
|
return { schemaVersion: DOCTOR_SCHEMA_VERSION, sections, exitCode };
|
|
4587
4846
|
}
|
|
4588
4847
|
export function runDoctor(options, dependencies = {}) {
|