@goondocks/myco 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/dist/{chunk-LDKXXKF6.js → chunk-2ZIBCEYO.js} +4 -4
- package/dist/{chunk-PQWQC3RF.js → chunk-4XVKZ3WA.js} +137 -146
- package/dist/chunk-4XVKZ3WA.js.map +1 -0
- package/dist/{chunk-MWW62YZP.js → chunk-7WHF2OIZ.js} +21 -19
- package/dist/chunk-7WHF2OIZ.js.map +1 -0
- package/dist/{chunk-JSK7L46L.js → chunk-ERG2IEWX.js} +22 -4
- package/dist/{chunk-JSK7L46L.js.map → chunk-ERG2IEWX.js.map} +1 -1
- package/dist/{chunk-RXJHB7W4.js → chunk-FPRXMJLT.js} +2 -2
- package/dist/{chunk-RY76WEN3.js → chunk-GENQ5QGP.js} +2 -2
- package/dist/{chunk-YG6MLLGL.js → chunk-HYVT345Y.js} +2 -2
- package/dist/{chunk-WBLTISAK.js → chunk-J4D4CROB.js} +32 -6
- package/dist/chunk-J4D4CROB.js.map +1 -0
- package/dist/{chunk-IWBWZQK6.js → chunk-MDLSAFPP.js} +2 -2
- package/dist/{chunk-HRGHDMYI.js → chunk-NL6WQO56.js} +2 -2
- package/dist/{chunk-V5R6O6RP.js → chunk-NLUE6CYG.js} +3 -3
- package/dist/{chunk-CQ4RKK67.js → chunk-O6PERU7U.js} +2 -2
- package/dist/{chunk-XNAM6Z4O.js → chunk-P723N2LP.js} +2 -2
- package/dist/{chunk-CK24O5YQ.js → chunk-QN4W3JUA.js} +2 -2
- package/dist/{chunk-ALBVNGCF.js → chunk-UP4P4OAA.js} +55 -44
- package/dist/{chunk-ALBVNGCF.js.map → chunk-UP4P4OAA.js.map} +1 -1
- package/dist/{chunk-CPVXNRGW.js → chunk-YIQLYIHW.js} +4 -4
- package/dist/{chunk-25DJSF2K.js → chunk-YTFXA4RX.js} +3 -3
- package/dist/{chunk-RNWALAFP.js → chunk-Z74SDEKE.js} +2 -2
- package/dist/chunk-Z74SDEKE.js.map +1 -0
- package/dist/{cli-LMBBPV2D.js → cli-IHILSS6N.js} +20 -20
- package/dist/{client-FDKJ4BY7.js → client-AGFNR2S4.js} +5 -5
- package/dist/{config-HDUFDOQN.js → config-IBS6KOLQ.js} +3 -3
- package/dist/{curate-DYE4VCBJ.js → curate-3D4GHKJH.js} +9 -10
- package/dist/{curate-DYE4VCBJ.js.map → curate-3D4GHKJH.js.map} +1 -1
- package/dist/{detect-providers-I2QQFDJW.js → detect-providers-XEP4QA3R.js} +3 -3
- package/dist/{digest-PNHFM7JJ.js → digest-7HLJXL77.js} +11 -11
- package/dist/{init-7N7F6W6U.js → init-ARQ53JOR.js} +8 -8
- package/dist/{main-3JZDUJLU.js → main-6AGPIMH2.js} +1972 -374
- package/dist/main-6AGPIMH2.js.map +1 -0
- package/dist/{rebuild-WXKQ5HZO.js → rebuild-Q2ACEB6F.js} +9 -10
- package/dist/{rebuild-WXKQ5HZO.js.map → rebuild-Q2ACEB6F.js.map} +1 -1
- package/dist/{reprocess-PKRDV67L.js → reprocess-CDEFGQOV.js} +11 -11
- package/dist/{restart-WSJRHRHI.js → restart-XCMILOL5.js} +6 -6
- package/dist/{search-SWMJ4MZ3.js → search-7W25SKCB.js} +6 -6
- package/dist/{server-NTRVB5ZM.js → server-6UDN35QN.js} +11 -11
- package/dist/{session-start-KQ4KCQMZ.js → session-start-K6IGAC7H.js} +9 -9
- package/dist/setup-digest-X5PN27F4.js +15 -0
- package/dist/setup-llm-S5OHQJXK.js +15 -0
- package/dist/src/cli.js +4 -4
- package/dist/src/daemon/main.js +4 -4
- package/dist/src/hooks/post-tool-use.js +5 -5
- package/dist/src/hooks/session-end.js +5 -5
- package/dist/src/hooks/session-start.js +4 -4
- package/dist/src/hooks/stop.js +7 -7
- package/dist/src/hooks/user-prompt-submit.js +5 -5
- package/dist/src/mcp/server.js +4 -4
- package/dist/src/prompts/extraction.md +4 -4
- package/dist/{stats-2OUQSEZO.js → stats-TTSDXGJV.js} +6 -6
- package/dist/ui/assets/index-08wKT7wS.css +1 -0
- package/dist/ui/assets/index-CMSMi4Jb.js +369 -0
- package/dist/ui/index.html +2 -2
- package/dist/{verify-MG5O7SBU.js → verify-TOWQHPBX.js} +6 -6
- package/dist/{version-NKOECSVH.js → version-36RVCQA6.js} +4 -4
- package/package.json +1 -1
- package/dist/chunk-MWW62YZP.js.map +0 -1
- package/dist/chunk-PQWQC3RF.js.map +0 -1
- package/dist/chunk-RNWALAFP.js.map +0 -1
- package/dist/chunk-WBLTISAK.js.map +0 -1
- package/dist/main-3JZDUJLU.js.map +0 -1
- package/dist/setup-digest-BOYOSM4B.js +0 -15
- package/dist/setup-llm-PCZ64ALK.js +0 -15
- package/dist/ui/assets/index-Bk4X_8-Z.css +0 -1
- package/dist/ui/assets/index-D3SY7ZHY.js +0 -299
- /package/dist/{chunk-LDKXXKF6.js.map → chunk-2ZIBCEYO.js.map} +0 -0
- /package/dist/{chunk-RXJHB7W4.js.map → chunk-FPRXMJLT.js.map} +0 -0
- /package/dist/{chunk-RY76WEN3.js.map → chunk-GENQ5QGP.js.map} +0 -0
- /package/dist/{chunk-YG6MLLGL.js.map → chunk-HYVT345Y.js.map} +0 -0
- /package/dist/{chunk-IWBWZQK6.js.map → chunk-MDLSAFPP.js.map} +0 -0
- /package/dist/{chunk-HRGHDMYI.js.map → chunk-NL6WQO56.js.map} +0 -0
- /package/dist/{chunk-V5R6O6RP.js.map → chunk-NLUE6CYG.js.map} +0 -0
- /package/dist/{chunk-CQ4RKK67.js.map → chunk-O6PERU7U.js.map} +0 -0
- /package/dist/{chunk-XNAM6Z4O.js.map → chunk-P723N2LP.js.map} +0 -0
- /package/dist/{chunk-CK24O5YQ.js.map → chunk-QN4W3JUA.js.map} +0 -0
- /package/dist/{chunk-CPVXNRGW.js.map → chunk-YIQLYIHW.js.map} +0 -0
- /package/dist/{chunk-25DJSF2K.js.map → chunk-YTFXA4RX.js.map} +0 -0
- /package/dist/{cli-LMBBPV2D.js.map → cli-IHILSS6N.js.map} +0 -0
- /package/dist/{client-FDKJ4BY7.js.map → client-AGFNR2S4.js.map} +0 -0
- /package/dist/{config-HDUFDOQN.js.map → config-IBS6KOLQ.js.map} +0 -0
- /package/dist/{detect-providers-I2QQFDJW.js.map → detect-providers-XEP4QA3R.js.map} +0 -0
- /package/dist/{digest-PNHFM7JJ.js.map → digest-7HLJXL77.js.map} +0 -0
- /package/dist/{init-7N7F6W6U.js.map → init-ARQ53JOR.js.map} +0 -0
- /package/dist/{reprocess-PKRDV67L.js.map → reprocess-CDEFGQOV.js.map} +0 -0
- /package/dist/{restart-WSJRHRHI.js.map → restart-XCMILOL5.js.map} +0 -0
- /package/dist/{search-SWMJ4MZ3.js.map → search-7W25SKCB.js.map} +0 -0
- /package/dist/{server-NTRVB5ZM.js.map → server-6UDN35QN.js.map} +0 -0
- /package/dist/{session-start-KQ4KCQMZ.js.map → session-start-K6IGAC7H.js.map} +0 -0
- /package/dist/{setup-digest-BOYOSM4B.js.map → setup-digest-X5PN27F4.js.map} +0 -0
- /package/dist/{setup-llm-PCZ64ALK.js.map → setup-llm-S5OHQJXK.js.map} +0 -0
- /package/dist/{stats-2OUQSEZO.js.map → stats-TTSDXGJV.js.map} +0 -0
- /package/dist/{verify-MG5O7SBU.js.map → verify-TOWQHPBX.js.map} +0 -0
- /package/dist/{version-NKOECSVH.js.map → version-36RVCQA6.js.map} +0 -0
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
import { createRequire as __cr } from 'node:module'; const require = __cr(import.meta.url);
|
|
2
2
|
import {
|
|
3
3
|
gatherStats
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-NL6WQO56.js";
|
|
5
5
|
import {
|
|
6
6
|
BufferProcessor,
|
|
7
7
|
DigestEngine,
|
|
8
8
|
Metabolism,
|
|
9
9
|
SUMMARIZATION_FAILED_MARKER,
|
|
10
|
-
TranscriptMiner,
|
|
11
10
|
appendTraceRecord,
|
|
12
|
-
|
|
11
|
+
readLastRecord,
|
|
13
12
|
readLastTimestamp,
|
|
14
13
|
runCuration,
|
|
15
14
|
runDigest,
|
|
16
15
|
runRebuild,
|
|
17
16
|
runReprocess,
|
|
17
|
+
updateTitleAndSummary,
|
|
18
18
|
writeObservationNotes
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-4XVKZ3WA.js";
|
|
20
20
|
import {
|
|
21
21
|
consolidateSpores,
|
|
22
22
|
handleMycoContext
|
|
23
|
-
} from "./chunk-
|
|
23
|
+
} from "./chunk-2ZIBCEYO.js";
|
|
24
24
|
import {
|
|
25
25
|
DaemonLogger
|
|
26
26
|
} from "./chunk-QLUE3BUL.js";
|
|
@@ -31,10 +31,9 @@ import {
|
|
|
31
31
|
CONVERSATION_HEADING,
|
|
32
32
|
VaultWriter,
|
|
33
33
|
bareSessionId,
|
|
34
|
-
buildSimilarityPrompt,
|
|
35
34
|
checkSupersession,
|
|
36
35
|
extractJson,
|
|
37
|
-
|
|
36
|
+
extractSection,
|
|
38
37
|
formatNotesForPrompt,
|
|
39
38
|
formatSessionBody,
|
|
40
39
|
indexNote,
|
|
@@ -44,51 +43,54 @@ import {
|
|
|
44
43
|
sessionNoteId,
|
|
45
44
|
sessionRelativePath,
|
|
46
45
|
sessionWikilink,
|
|
47
|
-
stripReasoningTokens
|
|
48
|
-
|
|
46
|
+
stripReasoningTokens,
|
|
47
|
+
walkMarkdownFiles
|
|
48
|
+
} from "./chunk-UP4P4OAA.js";
|
|
49
49
|
import {
|
|
50
50
|
generateEmbedding
|
|
51
51
|
} from "./chunk-RGVBGTD6.js";
|
|
52
52
|
import {
|
|
53
53
|
createEmbeddingProvider,
|
|
54
54
|
createLlmProvider
|
|
55
|
-
} from "./chunk-
|
|
56
|
-
import
|
|
55
|
+
} from "./chunk-NLUE6CYG.js";
|
|
56
|
+
import {
|
|
57
|
+
stripFrontmatter
|
|
58
|
+
} from "./chunk-GENQ5QGP.js";
|
|
57
59
|
import {
|
|
58
60
|
initFts
|
|
59
61
|
} from "./chunk-6FQISQNA.js";
|
|
60
62
|
import {
|
|
61
63
|
MycoIndex
|
|
62
64
|
} from "./chunk-TWSTAVLO.js";
|
|
63
|
-
import "./chunk-
|
|
65
|
+
import "./chunk-YTFXA4RX.js";
|
|
64
66
|
import "./chunk-SAKJMNSR.js";
|
|
65
67
|
import {
|
|
66
68
|
LmStudioBackend,
|
|
67
69
|
OllamaBackend
|
|
68
|
-
} from "./chunk-
|
|
70
|
+
} from "./chunk-7WHF2OIZ.js";
|
|
69
71
|
import {
|
|
70
72
|
CONFIG_FILENAME,
|
|
71
73
|
loadConfig,
|
|
72
74
|
saveConfig
|
|
73
|
-
} from "./chunk-
|
|
75
|
+
} from "./chunk-HYVT345Y.js";
|
|
74
76
|
import {
|
|
75
77
|
MycoConfigSchema,
|
|
76
78
|
external_exports,
|
|
77
79
|
require_dist
|
|
78
|
-
} from "./chunk-
|
|
80
|
+
} from "./chunk-ERG2IEWX.js";
|
|
79
81
|
import {
|
|
80
82
|
EventBuffer
|
|
81
83
|
} from "./chunk-HIN3UVOG.js";
|
|
82
84
|
import {
|
|
83
85
|
getPluginVersion
|
|
84
|
-
} from "./chunk-
|
|
86
|
+
} from "./chunk-QN4W3JUA.js";
|
|
85
87
|
import {
|
|
88
|
+
AgentRegistry,
|
|
86
89
|
claudeCodeAdapter,
|
|
87
90
|
createPerProjectAdapter,
|
|
88
91
|
extensionForMimeType
|
|
89
|
-
} from "./chunk-
|
|
92
|
+
} from "./chunk-Z74SDEKE.js";
|
|
90
93
|
import {
|
|
91
|
-
CANDIDATE_CONTENT_PREVIEW,
|
|
92
94
|
CONSOLIDATION_MAX_TOKENS,
|
|
93
95
|
CONSOLIDATION_MIN_CLUSTER_SIZE,
|
|
94
96
|
CONSOLIDATION_VECTOR_FETCH_LIMIT,
|
|
@@ -98,16 +100,27 @@ import {
|
|
|
98
100
|
DAEMON_EVICT_TIMEOUT_MS,
|
|
99
101
|
EMBEDDING_INPUT_LIMIT,
|
|
100
102
|
FILE_WATCH_STABILITY_MS,
|
|
103
|
+
ITEM_STAGE_MAP,
|
|
101
104
|
LINEAGE_RECENT_SESSIONS_LIMIT,
|
|
102
105
|
LLM_REASONING_MODE,
|
|
103
106
|
MAX_SLUG_LENGTH,
|
|
107
|
+
MS_PER_DAY,
|
|
108
|
+
PIPELINE_BACKOFF_MULTIPLIER,
|
|
109
|
+
PIPELINE_ITEMS_DEFAULT_LIMIT,
|
|
110
|
+
PIPELINE_PARSE_MAX_RETRIES,
|
|
111
|
+
PIPELINE_PROVIDER_ROLES,
|
|
112
|
+
PIPELINE_STAGES,
|
|
113
|
+
PIPELINE_TICK_STAGES,
|
|
104
114
|
PROMPT_CONTEXT_MAX_SPORES,
|
|
105
115
|
PROMPT_CONTEXT_MIN_LENGTH,
|
|
106
116
|
PROMPT_CONTEXT_MIN_SIMILARITY,
|
|
117
|
+
PROMPT_PREVIEW_CHARS,
|
|
107
118
|
RELATED_SPORES_LIMIT,
|
|
108
119
|
SESSION_CONTEXT_MAX_PLANS,
|
|
109
|
-
|
|
110
|
-
|
|
120
|
+
STAGE_PROVIDER_MAP,
|
|
121
|
+
STALE_BUFFER_MAX_AGE_MS,
|
|
122
|
+
estimateTokens
|
|
123
|
+
} from "./chunk-J4D4CROB.js";
|
|
111
124
|
import {
|
|
112
125
|
__toESM
|
|
113
126
|
} from "./chunk-PZUWP5VK.js";
|
|
@@ -502,10 +515,6 @@ import fs3 from "fs";
|
|
|
502
515
|
import path3 from "path";
|
|
503
516
|
var LINEAGE_IMMEDIATE_GAP_SECONDS = 5;
|
|
504
517
|
var LINEAGE_FALLBACK_MAX_HOURS = 24;
|
|
505
|
-
var LINEAGE_SIMILARITY_THRESHOLD = 0.7;
|
|
506
|
-
var LINEAGE_SIMILARITY_HIGH_CONFIDENCE = 0.9;
|
|
507
|
-
var LINEAGE_SIMILARITY_CANDIDATES = 3;
|
|
508
|
-
var LINEAGE_SIMILARITY_MAX_TOKENS = 8;
|
|
509
518
|
var MS_PER_SECOND = 1e3;
|
|
510
519
|
var MS_PER_HOUR = 36e5;
|
|
511
520
|
var LineageGraph = class {
|
|
@@ -697,7 +706,7 @@ var ReaddirpStream = class extends Readable {
|
|
|
697
706
|
this._directoryFilter = normalizeFilter(opts.directoryFilter);
|
|
698
707
|
const statMethod = opts.lstat ? lstat : stat;
|
|
699
708
|
if (wantBigintFsStats) {
|
|
700
|
-
this._stat = (
|
|
709
|
+
this._stat = (path12) => statMethod(path12, { bigint: true });
|
|
701
710
|
} else {
|
|
702
711
|
this._stat = statMethod;
|
|
703
712
|
}
|
|
@@ -722,8 +731,8 @@ var ReaddirpStream = class extends Readable {
|
|
|
722
731
|
const par = this.parent;
|
|
723
732
|
const fil = par && par.files;
|
|
724
733
|
if (fil && fil.length > 0) {
|
|
725
|
-
const { path:
|
|
726
|
-
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent,
|
|
734
|
+
const { path: path12, depth } = par;
|
|
735
|
+
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path12));
|
|
727
736
|
const awaited = await Promise.all(slice);
|
|
728
737
|
for (const entry of awaited) {
|
|
729
738
|
if (!entry)
|
|
@@ -763,20 +772,20 @@ var ReaddirpStream = class extends Readable {
|
|
|
763
772
|
this.reading = false;
|
|
764
773
|
}
|
|
765
774
|
}
|
|
766
|
-
async _exploreDir(
|
|
775
|
+
async _exploreDir(path12, depth) {
|
|
767
776
|
let files;
|
|
768
777
|
try {
|
|
769
|
-
files = await readdir(
|
|
778
|
+
files = await readdir(path12, this._rdOptions);
|
|
770
779
|
} catch (error) {
|
|
771
780
|
this._onError(error);
|
|
772
781
|
}
|
|
773
|
-
return { files, depth, path:
|
|
782
|
+
return { files, depth, path: path12 };
|
|
774
783
|
}
|
|
775
|
-
async _formatEntry(dirent,
|
|
784
|
+
async _formatEntry(dirent, path12) {
|
|
776
785
|
let entry;
|
|
777
786
|
const basename3 = this._isDirent ? dirent.name : dirent;
|
|
778
787
|
try {
|
|
779
|
-
const fullPath = presolve(pjoin(
|
|
788
|
+
const fullPath = presolve(pjoin(path12, basename3));
|
|
780
789
|
entry = { path: prelative(this._root, fullPath), fullPath, basename: basename3 };
|
|
781
790
|
entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
|
|
782
791
|
} catch (err) {
|
|
@@ -1176,16 +1185,16 @@ var delFromSet = (main2, prop, item) => {
|
|
|
1176
1185
|
};
|
|
1177
1186
|
var isEmptySet = (val) => val instanceof Set ? val.size === 0 : !val;
|
|
1178
1187
|
var FsWatchInstances = /* @__PURE__ */ new Map();
|
|
1179
|
-
function createFsWatchInstance(
|
|
1188
|
+
function createFsWatchInstance(path12, options, listener, errHandler, emitRaw) {
|
|
1180
1189
|
const handleEvent = (rawEvent, evPath) => {
|
|
1181
|
-
listener(
|
|
1182
|
-
emitRaw(rawEvent, evPath, { watchedPath:
|
|
1183
|
-
if (evPath &&
|
|
1184
|
-
fsWatchBroadcast(sp.resolve(
|
|
1190
|
+
listener(path12);
|
|
1191
|
+
emitRaw(rawEvent, evPath, { watchedPath: path12 });
|
|
1192
|
+
if (evPath && path12 !== evPath) {
|
|
1193
|
+
fsWatchBroadcast(sp.resolve(path12, evPath), KEY_LISTENERS, sp.join(path12, evPath));
|
|
1185
1194
|
}
|
|
1186
1195
|
};
|
|
1187
1196
|
try {
|
|
1188
|
-
return fs_watch(
|
|
1197
|
+
return fs_watch(path12, {
|
|
1189
1198
|
persistent: options.persistent
|
|
1190
1199
|
}, handleEvent);
|
|
1191
1200
|
} catch (error) {
|
|
@@ -1201,12 +1210,12 @@ var fsWatchBroadcast = (fullPath, listenerType, val1, val2, val3) => {
|
|
|
1201
1210
|
listener(val1, val2, val3);
|
|
1202
1211
|
});
|
|
1203
1212
|
};
|
|
1204
|
-
var setFsWatchListener = (
|
|
1213
|
+
var setFsWatchListener = (path12, fullPath, options, handlers) => {
|
|
1205
1214
|
const { listener, errHandler, rawEmitter } = handlers;
|
|
1206
1215
|
let cont = FsWatchInstances.get(fullPath);
|
|
1207
1216
|
let watcher;
|
|
1208
1217
|
if (!options.persistent) {
|
|
1209
|
-
watcher = createFsWatchInstance(
|
|
1218
|
+
watcher = createFsWatchInstance(path12, options, listener, errHandler, rawEmitter);
|
|
1210
1219
|
if (!watcher)
|
|
1211
1220
|
return;
|
|
1212
1221
|
return watcher.close.bind(watcher);
|
|
@@ -1217,7 +1226,7 @@ var setFsWatchListener = (path10, fullPath, options, handlers) => {
|
|
|
1217
1226
|
addAndConvert(cont, KEY_RAW, rawEmitter);
|
|
1218
1227
|
} else {
|
|
1219
1228
|
watcher = createFsWatchInstance(
|
|
1220
|
-
|
|
1229
|
+
path12,
|
|
1221
1230
|
options,
|
|
1222
1231
|
fsWatchBroadcast.bind(null, fullPath, KEY_LISTENERS),
|
|
1223
1232
|
errHandler,
|
|
@@ -1232,7 +1241,7 @@ var setFsWatchListener = (path10, fullPath, options, handlers) => {
|
|
|
1232
1241
|
cont.watcherUnusable = true;
|
|
1233
1242
|
if (isWindows && error.code === "EPERM") {
|
|
1234
1243
|
try {
|
|
1235
|
-
const fd = await open(
|
|
1244
|
+
const fd = await open(path12, "r");
|
|
1236
1245
|
await fd.close();
|
|
1237
1246
|
broadcastErr(error);
|
|
1238
1247
|
} catch (err) {
|
|
@@ -1263,7 +1272,7 @@ var setFsWatchListener = (path10, fullPath, options, handlers) => {
|
|
|
1263
1272
|
};
|
|
1264
1273
|
};
|
|
1265
1274
|
var FsWatchFileInstances = /* @__PURE__ */ new Map();
|
|
1266
|
-
var setFsWatchFileListener = (
|
|
1275
|
+
var setFsWatchFileListener = (path12, fullPath, options, handlers) => {
|
|
1267
1276
|
const { listener, rawEmitter } = handlers;
|
|
1268
1277
|
let cont = FsWatchFileInstances.get(fullPath);
|
|
1269
1278
|
const copts = cont && cont.options;
|
|
@@ -1285,7 +1294,7 @@ var setFsWatchFileListener = (path10, fullPath, options, handlers) => {
|
|
|
1285
1294
|
});
|
|
1286
1295
|
const currmtime = curr.mtimeMs;
|
|
1287
1296
|
if (curr.size !== prev.size || currmtime > prev.mtimeMs || currmtime === 0) {
|
|
1288
|
-
foreach(cont.listeners, (listener2) => listener2(
|
|
1297
|
+
foreach(cont.listeners, (listener2) => listener2(path12, curr));
|
|
1289
1298
|
}
|
|
1290
1299
|
})
|
|
1291
1300
|
};
|
|
@@ -1315,13 +1324,13 @@ var NodeFsHandler = class {
|
|
|
1315
1324
|
* @param listener on fs change
|
|
1316
1325
|
* @returns closer for the watcher instance
|
|
1317
1326
|
*/
|
|
1318
|
-
_watchWithNodeFs(
|
|
1327
|
+
_watchWithNodeFs(path12, listener) {
|
|
1319
1328
|
const opts = this.fsw.options;
|
|
1320
|
-
const directory = sp.dirname(
|
|
1321
|
-
const basename3 = sp.basename(
|
|
1329
|
+
const directory = sp.dirname(path12);
|
|
1330
|
+
const basename3 = sp.basename(path12);
|
|
1322
1331
|
const parent = this.fsw._getWatchedDir(directory);
|
|
1323
1332
|
parent.add(basename3);
|
|
1324
|
-
const absolutePath = sp.resolve(
|
|
1333
|
+
const absolutePath = sp.resolve(path12);
|
|
1325
1334
|
const options = {
|
|
1326
1335
|
persistent: opts.persistent
|
|
1327
1336
|
};
|
|
@@ -1331,12 +1340,12 @@ var NodeFsHandler = class {
|
|
|
1331
1340
|
if (opts.usePolling) {
|
|
1332
1341
|
const enableBin = opts.interval !== opts.binaryInterval;
|
|
1333
1342
|
options.interval = enableBin && isBinaryPath(basename3) ? opts.binaryInterval : opts.interval;
|
|
1334
|
-
closer = setFsWatchFileListener(
|
|
1343
|
+
closer = setFsWatchFileListener(path12, absolutePath, options, {
|
|
1335
1344
|
listener,
|
|
1336
1345
|
rawEmitter: this.fsw._emitRaw
|
|
1337
1346
|
});
|
|
1338
1347
|
} else {
|
|
1339
|
-
closer = setFsWatchListener(
|
|
1348
|
+
closer = setFsWatchListener(path12, absolutePath, options, {
|
|
1340
1349
|
listener,
|
|
1341
1350
|
errHandler: this._boundHandleError,
|
|
1342
1351
|
rawEmitter: this.fsw._emitRaw
|
|
@@ -1358,7 +1367,7 @@ var NodeFsHandler = class {
|
|
|
1358
1367
|
let prevStats = stats;
|
|
1359
1368
|
if (parent.has(basename3))
|
|
1360
1369
|
return;
|
|
1361
|
-
const listener = async (
|
|
1370
|
+
const listener = async (path12, newStats) => {
|
|
1362
1371
|
if (!this.fsw._throttle(THROTTLE_MODE_WATCH, file, 5))
|
|
1363
1372
|
return;
|
|
1364
1373
|
if (!newStats || newStats.mtimeMs === 0) {
|
|
@@ -1372,11 +1381,11 @@ var NodeFsHandler = class {
|
|
|
1372
1381
|
this.fsw._emit(EV.CHANGE, file, newStats2);
|
|
1373
1382
|
}
|
|
1374
1383
|
if ((isMacos || isLinux || isFreeBSD) && prevStats.ino !== newStats2.ino) {
|
|
1375
|
-
this.fsw._closeFile(
|
|
1384
|
+
this.fsw._closeFile(path12);
|
|
1376
1385
|
prevStats = newStats2;
|
|
1377
1386
|
const closer2 = this._watchWithNodeFs(file, listener);
|
|
1378
1387
|
if (closer2)
|
|
1379
|
-
this.fsw._addPathCloser(
|
|
1388
|
+
this.fsw._addPathCloser(path12, closer2);
|
|
1380
1389
|
} else {
|
|
1381
1390
|
prevStats = newStats2;
|
|
1382
1391
|
}
|
|
@@ -1408,7 +1417,7 @@ var NodeFsHandler = class {
|
|
|
1408
1417
|
* @param item basename of this item
|
|
1409
1418
|
* @returns true if no more processing is needed for this entry.
|
|
1410
1419
|
*/
|
|
1411
|
-
async _handleSymlink(entry, directory,
|
|
1420
|
+
async _handleSymlink(entry, directory, path12, item) {
|
|
1412
1421
|
if (this.fsw.closed) {
|
|
1413
1422
|
return;
|
|
1414
1423
|
}
|
|
@@ -1418,7 +1427,7 @@ var NodeFsHandler = class {
|
|
|
1418
1427
|
this.fsw._incrReadyCount();
|
|
1419
1428
|
let linkPath;
|
|
1420
1429
|
try {
|
|
1421
|
-
linkPath = await fsrealpath(
|
|
1430
|
+
linkPath = await fsrealpath(path12);
|
|
1422
1431
|
} catch (e) {
|
|
1423
1432
|
this.fsw._emitReady();
|
|
1424
1433
|
return true;
|
|
@@ -1428,12 +1437,12 @@ var NodeFsHandler = class {
|
|
|
1428
1437
|
if (dir.has(item)) {
|
|
1429
1438
|
if (this.fsw._symlinkPaths.get(full) !== linkPath) {
|
|
1430
1439
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
1431
|
-
this.fsw._emit(EV.CHANGE,
|
|
1440
|
+
this.fsw._emit(EV.CHANGE, path12, entry.stats);
|
|
1432
1441
|
}
|
|
1433
1442
|
} else {
|
|
1434
1443
|
dir.add(item);
|
|
1435
1444
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
1436
|
-
this.fsw._emit(EV.ADD,
|
|
1445
|
+
this.fsw._emit(EV.ADD, path12, entry.stats);
|
|
1437
1446
|
}
|
|
1438
1447
|
this.fsw._emitReady();
|
|
1439
1448
|
return true;
|
|
@@ -1463,9 +1472,9 @@ var NodeFsHandler = class {
|
|
|
1463
1472
|
return;
|
|
1464
1473
|
}
|
|
1465
1474
|
const item = entry.path;
|
|
1466
|
-
let
|
|
1475
|
+
let path12 = sp.join(directory, item);
|
|
1467
1476
|
current.add(item);
|
|
1468
|
-
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory,
|
|
1477
|
+
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory, path12, item)) {
|
|
1469
1478
|
return;
|
|
1470
1479
|
}
|
|
1471
1480
|
if (this.fsw.closed) {
|
|
@@ -1474,8 +1483,8 @@ var NodeFsHandler = class {
|
|
|
1474
1483
|
}
|
|
1475
1484
|
if (item === target || !target && !previous.has(item)) {
|
|
1476
1485
|
this.fsw._incrReadyCount();
|
|
1477
|
-
|
|
1478
|
-
this._addToNodeFs(
|
|
1486
|
+
path12 = sp.join(dir, sp.relative(dir, path12));
|
|
1487
|
+
this._addToNodeFs(path12, initialAdd, wh, depth + 1);
|
|
1479
1488
|
}
|
|
1480
1489
|
}).on(EV.ERROR, this._boundHandleError);
|
|
1481
1490
|
return new Promise((resolve3, reject) => {
|
|
@@ -1544,13 +1553,13 @@ var NodeFsHandler = class {
|
|
|
1544
1553
|
* @param depth Child path actually targeted for watch
|
|
1545
1554
|
* @param target Child path actually targeted for watch
|
|
1546
1555
|
*/
|
|
1547
|
-
async _addToNodeFs(
|
|
1556
|
+
async _addToNodeFs(path12, initialAdd, priorWh, depth, target) {
|
|
1548
1557
|
const ready = this.fsw._emitReady;
|
|
1549
|
-
if (this.fsw._isIgnored(
|
|
1558
|
+
if (this.fsw._isIgnored(path12) || this.fsw.closed) {
|
|
1550
1559
|
ready();
|
|
1551
1560
|
return false;
|
|
1552
1561
|
}
|
|
1553
|
-
const wh = this.fsw._getWatchHelpers(
|
|
1562
|
+
const wh = this.fsw._getWatchHelpers(path12);
|
|
1554
1563
|
if (priorWh) {
|
|
1555
1564
|
wh.filterPath = (entry) => priorWh.filterPath(entry);
|
|
1556
1565
|
wh.filterDir = (entry) => priorWh.filterDir(entry);
|
|
@@ -1566,8 +1575,8 @@ var NodeFsHandler = class {
|
|
|
1566
1575
|
const follow = this.fsw.options.followSymlinks;
|
|
1567
1576
|
let closer;
|
|
1568
1577
|
if (stats.isDirectory()) {
|
|
1569
|
-
const absPath = sp.resolve(
|
|
1570
|
-
const targetPath = follow ? await fsrealpath(
|
|
1578
|
+
const absPath = sp.resolve(path12);
|
|
1579
|
+
const targetPath = follow ? await fsrealpath(path12) : path12;
|
|
1571
1580
|
if (this.fsw.closed)
|
|
1572
1581
|
return;
|
|
1573
1582
|
closer = await this._handleDir(wh.watchPath, stats, initialAdd, depth, target, wh, targetPath);
|
|
@@ -1577,29 +1586,29 @@ var NodeFsHandler = class {
|
|
|
1577
1586
|
this.fsw._symlinkPaths.set(absPath, targetPath);
|
|
1578
1587
|
}
|
|
1579
1588
|
} else if (stats.isSymbolicLink()) {
|
|
1580
|
-
const targetPath = follow ? await fsrealpath(
|
|
1589
|
+
const targetPath = follow ? await fsrealpath(path12) : path12;
|
|
1581
1590
|
if (this.fsw.closed)
|
|
1582
1591
|
return;
|
|
1583
1592
|
const parent = sp.dirname(wh.watchPath);
|
|
1584
1593
|
this.fsw._getWatchedDir(parent).add(wh.watchPath);
|
|
1585
1594
|
this.fsw._emit(EV.ADD, wh.watchPath, stats);
|
|
1586
|
-
closer = await this._handleDir(parent, stats, initialAdd, depth,
|
|
1595
|
+
closer = await this._handleDir(parent, stats, initialAdd, depth, path12, wh, targetPath);
|
|
1587
1596
|
if (this.fsw.closed)
|
|
1588
1597
|
return;
|
|
1589
1598
|
if (targetPath !== void 0) {
|
|
1590
|
-
this.fsw._symlinkPaths.set(sp.resolve(
|
|
1599
|
+
this.fsw._symlinkPaths.set(sp.resolve(path12), targetPath);
|
|
1591
1600
|
}
|
|
1592
1601
|
} else {
|
|
1593
1602
|
closer = this._handleFile(wh.watchPath, stats, initialAdd);
|
|
1594
1603
|
}
|
|
1595
1604
|
ready();
|
|
1596
1605
|
if (closer)
|
|
1597
|
-
this.fsw._addPathCloser(
|
|
1606
|
+
this.fsw._addPathCloser(path12, closer);
|
|
1598
1607
|
return false;
|
|
1599
1608
|
} catch (error) {
|
|
1600
1609
|
if (this.fsw._handleError(error)) {
|
|
1601
1610
|
ready();
|
|
1602
|
-
return
|
|
1611
|
+
return path12;
|
|
1603
1612
|
}
|
|
1604
1613
|
}
|
|
1605
1614
|
}
|
|
@@ -1642,24 +1651,24 @@ function createPattern(matcher) {
|
|
|
1642
1651
|
}
|
|
1643
1652
|
return () => false;
|
|
1644
1653
|
}
|
|
1645
|
-
function normalizePath(
|
|
1646
|
-
if (typeof
|
|
1654
|
+
function normalizePath(path12) {
|
|
1655
|
+
if (typeof path12 !== "string")
|
|
1647
1656
|
throw new Error("string expected");
|
|
1648
|
-
|
|
1649
|
-
|
|
1657
|
+
path12 = sp2.normalize(path12);
|
|
1658
|
+
path12 = path12.replace(/\\/g, "/");
|
|
1650
1659
|
let prepend = false;
|
|
1651
|
-
if (
|
|
1660
|
+
if (path12.startsWith("//"))
|
|
1652
1661
|
prepend = true;
|
|
1653
|
-
|
|
1662
|
+
path12 = path12.replace(DOUBLE_SLASH_RE, "/");
|
|
1654
1663
|
if (prepend)
|
|
1655
|
-
|
|
1656
|
-
return
|
|
1664
|
+
path12 = "/" + path12;
|
|
1665
|
+
return path12;
|
|
1657
1666
|
}
|
|
1658
1667
|
function matchPatterns(patterns, testString, stats) {
|
|
1659
|
-
const
|
|
1668
|
+
const path12 = normalizePath(testString);
|
|
1660
1669
|
for (let index = 0; index < patterns.length; index++) {
|
|
1661
1670
|
const pattern = patterns[index];
|
|
1662
|
-
if (pattern(
|
|
1671
|
+
if (pattern(path12, stats)) {
|
|
1663
1672
|
return true;
|
|
1664
1673
|
}
|
|
1665
1674
|
}
|
|
@@ -1697,19 +1706,19 @@ var toUnix = (string) => {
|
|
|
1697
1706
|
}
|
|
1698
1707
|
return str;
|
|
1699
1708
|
};
|
|
1700
|
-
var normalizePathToUnix = (
|
|
1701
|
-
var normalizeIgnored = (cwd = "") => (
|
|
1702
|
-
if (typeof
|
|
1703
|
-
return normalizePathToUnix(sp2.isAbsolute(
|
|
1709
|
+
var normalizePathToUnix = (path12) => toUnix(sp2.normalize(toUnix(path12)));
|
|
1710
|
+
var normalizeIgnored = (cwd = "") => (path12) => {
|
|
1711
|
+
if (typeof path12 === "string") {
|
|
1712
|
+
return normalizePathToUnix(sp2.isAbsolute(path12) ? path12 : sp2.join(cwd, path12));
|
|
1704
1713
|
} else {
|
|
1705
|
-
return
|
|
1714
|
+
return path12;
|
|
1706
1715
|
}
|
|
1707
1716
|
};
|
|
1708
|
-
var getAbsolutePath = (
|
|
1709
|
-
if (sp2.isAbsolute(
|
|
1710
|
-
return
|
|
1717
|
+
var getAbsolutePath = (path12, cwd) => {
|
|
1718
|
+
if (sp2.isAbsolute(path12)) {
|
|
1719
|
+
return path12;
|
|
1711
1720
|
}
|
|
1712
|
-
return sp2.join(cwd,
|
|
1721
|
+
return sp2.join(cwd, path12);
|
|
1713
1722
|
};
|
|
1714
1723
|
var EMPTY_SET = Object.freeze(/* @__PURE__ */ new Set());
|
|
1715
1724
|
var DirEntry = class {
|
|
@@ -1774,10 +1783,10 @@ var WatchHelper = class {
|
|
|
1774
1783
|
dirParts;
|
|
1775
1784
|
followSymlinks;
|
|
1776
1785
|
statMethod;
|
|
1777
|
-
constructor(
|
|
1786
|
+
constructor(path12, follow, fsw) {
|
|
1778
1787
|
this.fsw = fsw;
|
|
1779
|
-
const watchPath =
|
|
1780
|
-
this.path =
|
|
1788
|
+
const watchPath = path12;
|
|
1789
|
+
this.path = path12 = path12.replace(REPLACER_RE, "");
|
|
1781
1790
|
this.watchPath = watchPath;
|
|
1782
1791
|
this.fullWatchPath = sp2.resolve(watchPath);
|
|
1783
1792
|
this.dirParts = [];
|
|
@@ -1917,20 +1926,20 @@ var FSWatcher = class extends EventEmitter {
|
|
|
1917
1926
|
this._closePromise = void 0;
|
|
1918
1927
|
let paths = unifyPaths(paths_);
|
|
1919
1928
|
if (cwd) {
|
|
1920
|
-
paths = paths.map((
|
|
1921
|
-
const absPath = getAbsolutePath(
|
|
1929
|
+
paths = paths.map((path12) => {
|
|
1930
|
+
const absPath = getAbsolutePath(path12, cwd);
|
|
1922
1931
|
return absPath;
|
|
1923
1932
|
});
|
|
1924
1933
|
}
|
|
1925
|
-
paths.forEach((
|
|
1926
|
-
this._removeIgnoredPath(
|
|
1934
|
+
paths.forEach((path12) => {
|
|
1935
|
+
this._removeIgnoredPath(path12);
|
|
1927
1936
|
});
|
|
1928
1937
|
this._userIgnored = void 0;
|
|
1929
1938
|
if (!this._readyCount)
|
|
1930
1939
|
this._readyCount = 0;
|
|
1931
1940
|
this._readyCount += paths.length;
|
|
1932
|
-
Promise.all(paths.map(async (
|
|
1933
|
-
const res = await this._nodeFsHandler._addToNodeFs(
|
|
1941
|
+
Promise.all(paths.map(async (path12) => {
|
|
1942
|
+
const res = await this._nodeFsHandler._addToNodeFs(path12, !_internal, void 0, 0, _origAdd);
|
|
1934
1943
|
if (res)
|
|
1935
1944
|
this._emitReady();
|
|
1936
1945
|
return res;
|
|
@@ -1952,17 +1961,17 @@ var FSWatcher = class extends EventEmitter {
|
|
|
1952
1961
|
return this;
|
|
1953
1962
|
const paths = unifyPaths(paths_);
|
|
1954
1963
|
const { cwd } = this.options;
|
|
1955
|
-
paths.forEach((
|
|
1956
|
-
if (!sp2.isAbsolute(
|
|
1964
|
+
paths.forEach((path12) => {
|
|
1965
|
+
if (!sp2.isAbsolute(path12) && !this._closers.has(path12)) {
|
|
1957
1966
|
if (cwd)
|
|
1958
|
-
|
|
1959
|
-
|
|
1967
|
+
path12 = sp2.join(cwd, path12);
|
|
1968
|
+
path12 = sp2.resolve(path12);
|
|
1960
1969
|
}
|
|
1961
|
-
this._closePath(
|
|
1962
|
-
this._addIgnoredPath(
|
|
1963
|
-
if (this._watched.has(
|
|
1970
|
+
this._closePath(path12);
|
|
1971
|
+
this._addIgnoredPath(path12);
|
|
1972
|
+
if (this._watched.has(path12)) {
|
|
1964
1973
|
this._addIgnoredPath({
|
|
1965
|
-
path:
|
|
1974
|
+
path: path12,
|
|
1966
1975
|
recursive: true
|
|
1967
1976
|
});
|
|
1968
1977
|
}
|
|
@@ -2026,38 +2035,38 @@ var FSWatcher = class extends EventEmitter {
|
|
|
2026
2035
|
* @param stats arguments to be passed with event
|
|
2027
2036
|
* @returns the error if defined, otherwise the value of the FSWatcher instance's `closed` flag
|
|
2028
2037
|
*/
|
|
2029
|
-
async _emit(event,
|
|
2038
|
+
async _emit(event, path12, stats) {
|
|
2030
2039
|
if (this.closed)
|
|
2031
2040
|
return;
|
|
2032
2041
|
const opts = this.options;
|
|
2033
2042
|
if (isWindows)
|
|
2034
|
-
|
|
2043
|
+
path12 = sp2.normalize(path12);
|
|
2035
2044
|
if (opts.cwd)
|
|
2036
|
-
|
|
2037
|
-
const args = [
|
|
2045
|
+
path12 = sp2.relative(opts.cwd, path12);
|
|
2046
|
+
const args = [path12];
|
|
2038
2047
|
if (stats != null)
|
|
2039
2048
|
args.push(stats);
|
|
2040
2049
|
const awf = opts.awaitWriteFinish;
|
|
2041
2050
|
let pw;
|
|
2042
|
-
if (awf && (pw = this._pendingWrites.get(
|
|
2051
|
+
if (awf && (pw = this._pendingWrites.get(path12))) {
|
|
2043
2052
|
pw.lastChange = /* @__PURE__ */ new Date();
|
|
2044
2053
|
return this;
|
|
2045
2054
|
}
|
|
2046
2055
|
if (opts.atomic) {
|
|
2047
2056
|
if (event === EVENTS.UNLINK) {
|
|
2048
|
-
this._pendingUnlinks.set(
|
|
2057
|
+
this._pendingUnlinks.set(path12, [event, ...args]);
|
|
2049
2058
|
setTimeout(() => {
|
|
2050
|
-
this._pendingUnlinks.forEach((entry,
|
|
2059
|
+
this._pendingUnlinks.forEach((entry, path13) => {
|
|
2051
2060
|
this.emit(...entry);
|
|
2052
2061
|
this.emit(EVENTS.ALL, ...entry);
|
|
2053
|
-
this._pendingUnlinks.delete(
|
|
2062
|
+
this._pendingUnlinks.delete(path13);
|
|
2054
2063
|
});
|
|
2055
2064
|
}, typeof opts.atomic === "number" ? opts.atomic : 100);
|
|
2056
2065
|
return this;
|
|
2057
2066
|
}
|
|
2058
|
-
if (event === EVENTS.ADD && this._pendingUnlinks.has(
|
|
2067
|
+
if (event === EVENTS.ADD && this._pendingUnlinks.has(path12)) {
|
|
2059
2068
|
event = EVENTS.CHANGE;
|
|
2060
|
-
this._pendingUnlinks.delete(
|
|
2069
|
+
this._pendingUnlinks.delete(path12);
|
|
2061
2070
|
}
|
|
2062
2071
|
}
|
|
2063
2072
|
if (awf && (event === EVENTS.ADD || event === EVENTS.CHANGE) && this._readyEmitted) {
|
|
@@ -2075,16 +2084,16 @@ var FSWatcher = class extends EventEmitter {
|
|
|
2075
2084
|
this.emitWithAll(event, args);
|
|
2076
2085
|
}
|
|
2077
2086
|
};
|
|
2078
|
-
this._awaitWriteFinish(
|
|
2087
|
+
this._awaitWriteFinish(path12, awf.stabilityThreshold, event, awfEmit);
|
|
2079
2088
|
return this;
|
|
2080
2089
|
}
|
|
2081
2090
|
if (event === EVENTS.CHANGE) {
|
|
2082
|
-
const isThrottled = !this._throttle(EVENTS.CHANGE,
|
|
2091
|
+
const isThrottled = !this._throttle(EVENTS.CHANGE, path12, 50);
|
|
2083
2092
|
if (isThrottled)
|
|
2084
2093
|
return this;
|
|
2085
2094
|
}
|
|
2086
2095
|
if (opts.alwaysStat && stats === void 0 && (event === EVENTS.ADD || event === EVENTS.ADD_DIR || event === EVENTS.CHANGE)) {
|
|
2087
|
-
const fullPath = opts.cwd ? sp2.join(opts.cwd,
|
|
2096
|
+
const fullPath = opts.cwd ? sp2.join(opts.cwd, path12) : path12;
|
|
2088
2097
|
let stats2;
|
|
2089
2098
|
try {
|
|
2090
2099
|
stats2 = await stat3(fullPath);
|
|
@@ -2115,23 +2124,23 @@ var FSWatcher = class extends EventEmitter {
|
|
|
2115
2124
|
* @param timeout duration of time to suppress duplicate actions
|
|
2116
2125
|
* @returns tracking object or false if action should be suppressed
|
|
2117
2126
|
*/
|
|
2118
|
-
_throttle(actionType,
|
|
2127
|
+
_throttle(actionType, path12, timeout) {
|
|
2119
2128
|
if (!this._throttled.has(actionType)) {
|
|
2120
2129
|
this._throttled.set(actionType, /* @__PURE__ */ new Map());
|
|
2121
2130
|
}
|
|
2122
2131
|
const action = this._throttled.get(actionType);
|
|
2123
2132
|
if (!action)
|
|
2124
2133
|
throw new Error("invalid throttle");
|
|
2125
|
-
const actionPath = action.get(
|
|
2134
|
+
const actionPath = action.get(path12);
|
|
2126
2135
|
if (actionPath) {
|
|
2127
2136
|
actionPath.count++;
|
|
2128
2137
|
return false;
|
|
2129
2138
|
}
|
|
2130
2139
|
let timeoutObject;
|
|
2131
2140
|
const clear = () => {
|
|
2132
|
-
const item = action.get(
|
|
2141
|
+
const item = action.get(path12);
|
|
2133
2142
|
const count = item ? item.count : 0;
|
|
2134
|
-
action.delete(
|
|
2143
|
+
action.delete(path12);
|
|
2135
2144
|
clearTimeout(timeoutObject);
|
|
2136
2145
|
if (item)
|
|
2137
2146
|
clearTimeout(item.timeoutObject);
|
|
@@ -2139,7 +2148,7 @@ var FSWatcher = class extends EventEmitter {
|
|
|
2139
2148
|
};
|
|
2140
2149
|
timeoutObject = setTimeout(clear, timeout);
|
|
2141
2150
|
const thr = { timeoutObject, clear, count: 0 };
|
|
2142
|
-
action.set(
|
|
2151
|
+
action.set(path12, thr);
|
|
2143
2152
|
return thr;
|
|
2144
2153
|
}
|
|
2145
2154
|
_incrReadyCount() {
|
|
@@ -2153,44 +2162,44 @@ var FSWatcher = class extends EventEmitter {
|
|
|
2153
2162
|
* @param event
|
|
2154
2163
|
* @param awfEmit Callback to be called when ready for event to be emitted.
|
|
2155
2164
|
*/
|
|
2156
|
-
_awaitWriteFinish(
|
|
2165
|
+
_awaitWriteFinish(path12, threshold, event, awfEmit) {
|
|
2157
2166
|
const awf = this.options.awaitWriteFinish;
|
|
2158
2167
|
if (typeof awf !== "object")
|
|
2159
2168
|
return;
|
|
2160
2169
|
const pollInterval = awf.pollInterval;
|
|
2161
2170
|
let timeoutHandler;
|
|
2162
|
-
let fullPath =
|
|
2163
|
-
if (this.options.cwd && !sp2.isAbsolute(
|
|
2164
|
-
fullPath = sp2.join(this.options.cwd,
|
|
2171
|
+
let fullPath = path12;
|
|
2172
|
+
if (this.options.cwd && !sp2.isAbsolute(path12)) {
|
|
2173
|
+
fullPath = sp2.join(this.options.cwd, path12);
|
|
2165
2174
|
}
|
|
2166
2175
|
const now = /* @__PURE__ */ new Date();
|
|
2167
2176
|
const writes = this._pendingWrites;
|
|
2168
2177
|
function awaitWriteFinishFn(prevStat) {
|
|
2169
2178
|
statcb(fullPath, (err, curStat) => {
|
|
2170
|
-
if (err || !writes.has(
|
|
2179
|
+
if (err || !writes.has(path12)) {
|
|
2171
2180
|
if (err && err.code !== "ENOENT")
|
|
2172
2181
|
awfEmit(err);
|
|
2173
2182
|
return;
|
|
2174
2183
|
}
|
|
2175
2184
|
const now2 = Number(/* @__PURE__ */ new Date());
|
|
2176
2185
|
if (prevStat && curStat.size !== prevStat.size) {
|
|
2177
|
-
writes.get(
|
|
2186
|
+
writes.get(path12).lastChange = now2;
|
|
2178
2187
|
}
|
|
2179
|
-
const pw = writes.get(
|
|
2188
|
+
const pw = writes.get(path12);
|
|
2180
2189
|
const df = now2 - pw.lastChange;
|
|
2181
2190
|
if (df >= threshold) {
|
|
2182
|
-
writes.delete(
|
|
2191
|
+
writes.delete(path12);
|
|
2183
2192
|
awfEmit(void 0, curStat);
|
|
2184
2193
|
} else {
|
|
2185
2194
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval, curStat);
|
|
2186
2195
|
}
|
|
2187
2196
|
});
|
|
2188
2197
|
}
|
|
2189
|
-
if (!writes.has(
|
|
2190
|
-
writes.set(
|
|
2198
|
+
if (!writes.has(path12)) {
|
|
2199
|
+
writes.set(path12, {
|
|
2191
2200
|
lastChange: now,
|
|
2192
2201
|
cancelWait: () => {
|
|
2193
|
-
writes.delete(
|
|
2202
|
+
writes.delete(path12);
|
|
2194
2203
|
clearTimeout(timeoutHandler);
|
|
2195
2204
|
return event;
|
|
2196
2205
|
}
|
|
@@ -2201,8 +2210,8 @@ var FSWatcher = class extends EventEmitter {
|
|
|
2201
2210
|
/**
|
|
2202
2211
|
* Determines whether user has asked to ignore this path.
|
|
2203
2212
|
*/
|
|
2204
|
-
_isIgnored(
|
|
2205
|
-
if (this.options.atomic && DOT_RE.test(
|
|
2213
|
+
_isIgnored(path12, stats) {
|
|
2214
|
+
if (this.options.atomic && DOT_RE.test(path12))
|
|
2206
2215
|
return true;
|
|
2207
2216
|
if (!this._userIgnored) {
|
|
2208
2217
|
const { cwd } = this.options;
|
|
@@ -2212,17 +2221,17 @@ var FSWatcher = class extends EventEmitter {
|
|
|
2212
2221
|
const list = [...ignoredPaths.map(normalizeIgnored(cwd)), ...ignored];
|
|
2213
2222
|
this._userIgnored = anymatch(list, void 0);
|
|
2214
2223
|
}
|
|
2215
|
-
return this._userIgnored(
|
|
2224
|
+
return this._userIgnored(path12, stats);
|
|
2216
2225
|
}
|
|
2217
|
-
_isntIgnored(
|
|
2218
|
-
return !this._isIgnored(
|
|
2226
|
+
_isntIgnored(path12, stat4) {
|
|
2227
|
+
return !this._isIgnored(path12, stat4);
|
|
2219
2228
|
}
|
|
2220
2229
|
/**
|
|
2221
2230
|
* Provides a set of common helpers and properties relating to symlink handling.
|
|
2222
2231
|
* @param path file or directory pattern being watched
|
|
2223
2232
|
*/
|
|
2224
|
-
_getWatchHelpers(
|
|
2225
|
-
return new WatchHelper(
|
|
2233
|
+
_getWatchHelpers(path12) {
|
|
2234
|
+
return new WatchHelper(path12, this.options.followSymlinks, this);
|
|
2226
2235
|
}
|
|
2227
2236
|
// Directory helpers
|
|
2228
2237
|
// -----------------
|
|
@@ -2254,63 +2263,63 @@ var FSWatcher = class extends EventEmitter {
|
|
|
2254
2263
|
* @param item base path of item/directory
|
|
2255
2264
|
*/
|
|
2256
2265
|
_remove(directory, item, isDirectory) {
|
|
2257
|
-
const
|
|
2258
|
-
const fullPath = sp2.resolve(
|
|
2259
|
-
isDirectory = isDirectory != null ? isDirectory : this._watched.has(
|
|
2260
|
-
if (!this._throttle("remove",
|
|
2266
|
+
const path12 = sp2.join(directory, item);
|
|
2267
|
+
const fullPath = sp2.resolve(path12);
|
|
2268
|
+
isDirectory = isDirectory != null ? isDirectory : this._watched.has(path12) || this._watched.has(fullPath);
|
|
2269
|
+
if (!this._throttle("remove", path12, 100))
|
|
2261
2270
|
return;
|
|
2262
2271
|
if (!isDirectory && this._watched.size === 1) {
|
|
2263
2272
|
this.add(directory, item, true);
|
|
2264
2273
|
}
|
|
2265
|
-
const wp = this._getWatchedDir(
|
|
2274
|
+
const wp = this._getWatchedDir(path12);
|
|
2266
2275
|
const nestedDirectoryChildren = wp.getChildren();
|
|
2267
|
-
nestedDirectoryChildren.forEach((nested) => this._remove(
|
|
2276
|
+
nestedDirectoryChildren.forEach((nested) => this._remove(path12, nested));
|
|
2268
2277
|
const parent = this._getWatchedDir(directory);
|
|
2269
2278
|
const wasTracked = parent.has(item);
|
|
2270
2279
|
parent.remove(item);
|
|
2271
2280
|
if (this._symlinkPaths.has(fullPath)) {
|
|
2272
2281
|
this._symlinkPaths.delete(fullPath);
|
|
2273
2282
|
}
|
|
2274
|
-
let relPath =
|
|
2283
|
+
let relPath = path12;
|
|
2275
2284
|
if (this.options.cwd)
|
|
2276
|
-
relPath = sp2.relative(this.options.cwd,
|
|
2285
|
+
relPath = sp2.relative(this.options.cwd, path12);
|
|
2277
2286
|
if (this.options.awaitWriteFinish && this._pendingWrites.has(relPath)) {
|
|
2278
2287
|
const event = this._pendingWrites.get(relPath).cancelWait();
|
|
2279
2288
|
if (event === EVENTS.ADD)
|
|
2280
2289
|
return;
|
|
2281
2290
|
}
|
|
2282
|
-
this._watched.delete(
|
|
2291
|
+
this._watched.delete(path12);
|
|
2283
2292
|
this._watched.delete(fullPath);
|
|
2284
2293
|
const eventName = isDirectory ? EVENTS.UNLINK_DIR : EVENTS.UNLINK;
|
|
2285
|
-
if (wasTracked && !this._isIgnored(
|
|
2286
|
-
this._emit(eventName,
|
|
2287
|
-
this._closePath(
|
|
2294
|
+
if (wasTracked && !this._isIgnored(path12))
|
|
2295
|
+
this._emit(eventName, path12);
|
|
2296
|
+
this._closePath(path12);
|
|
2288
2297
|
}
|
|
2289
2298
|
/**
|
|
2290
2299
|
* Closes all watchers for a path
|
|
2291
2300
|
*/
|
|
2292
|
-
_closePath(
|
|
2293
|
-
this._closeFile(
|
|
2294
|
-
const dir = sp2.dirname(
|
|
2295
|
-
this._getWatchedDir(dir).remove(sp2.basename(
|
|
2301
|
+
_closePath(path12) {
|
|
2302
|
+
this._closeFile(path12);
|
|
2303
|
+
const dir = sp2.dirname(path12);
|
|
2304
|
+
this._getWatchedDir(dir).remove(sp2.basename(path12));
|
|
2296
2305
|
}
|
|
2297
2306
|
/**
|
|
2298
2307
|
* Closes only file-specific watchers
|
|
2299
2308
|
*/
|
|
2300
|
-
_closeFile(
|
|
2301
|
-
const closers = this._closers.get(
|
|
2309
|
+
_closeFile(path12) {
|
|
2310
|
+
const closers = this._closers.get(path12);
|
|
2302
2311
|
if (!closers)
|
|
2303
2312
|
return;
|
|
2304
2313
|
closers.forEach((closer) => closer());
|
|
2305
|
-
this._closers.delete(
|
|
2314
|
+
this._closers.delete(path12);
|
|
2306
2315
|
}
|
|
2307
|
-
_addPathCloser(
|
|
2316
|
+
_addPathCloser(path12, closer) {
|
|
2308
2317
|
if (!closer)
|
|
2309
2318
|
return;
|
|
2310
|
-
let list = this._closers.get(
|
|
2319
|
+
let list = this._closers.get(path12);
|
|
2311
2320
|
if (!list) {
|
|
2312
2321
|
list = [];
|
|
2313
|
-
this._closers.set(
|
|
2322
|
+
this._closers.set(path12, list);
|
|
2314
2323
|
}
|
|
2315
2324
|
list.push(closer);
|
|
2316
2325
|
}
|
|
@@ -2690,6 +2699,53 @@ function isPortAvailable(port) {
|
|
|
2690
2699
|
});
|
|
2691
2700
|
}
|
|
2692
2701
|
|
|
2702
|
+
// src/capture/transcript-miner.ts
|
|
2703
|
+
var TranscriptMiner = class {
|
|
2704
|
+
registry;
|
|
2705
|
+
constructor(config) {
|
|
2706
|
+
this.registry = new AgentRegistry(config?.additionalAdapters);
|
|
2707
|
+
}
|
|
2708
|
+
/**
|
|
2709
|
+
* Extract all conversation turns for a session.
|
|
2710
|
+
* Convenience wrapper — delegates to getAllTurnsWithSource.
|
|
2711
|
+
*/
|
|
2712
|
+
getAllTurns(sessionId) {
|
|
2713
|
+
return this.getAllTurnsWithSource(sessionId).turns;
|
|
2714
|
+
}
|
|
2715
|
+
/**
|
|
2716
|
+
* Extract turns using the hook-provided transcript path first (fast, no scanning),
|
|
2717
|
+
* then fall back to adapter registry scanning if the path isn't provided.
|
|
2718
|
+
*/
|
|
2719
|
+
getAllTurnsWithSource(sessionId, transcriptPath) {
|
|
2720
|
+
if (transcriptPath) {
|
|
2721
|
+
const result2 = this.registry.parseTurnsFromPath(transcriptPath);
|
|
2722
|
+
if (result2) return result2;
|
|
2723
|
+
}
|
|
2724
|
+
const result = this.registry.getTranscriptTurns(sessionId);
|
|
2725
|
+
if (result) return result;
|
|
2726
|
+
return { turns: [], source: "none" };
|
|
2727
|
+
}
|
|
2728
|
+
};
|
|
2729
|
+
function extractTurnsFromBuffer(events) {
|
|
2730
|
+
const turns = [];
|
|
2731
|
+
let current = null;
|
|
2732
|
+
for (const event of events) {
|
|
2733
|
+
const type = event.type;
|
|
2734
|
+
if (type === "user_prompt") {
|
|
2735
|
+
if (current) turns.push(current);
|
|
2736
|
+
current = {
|
|
2737
|
+
prompt: String(event.prompt ?? "").slice(0, PROMPT_PREVIEW_CHARS),
|
|
2738
|
+
toolCount: 0,
|
|
2739
|
+
timestamp: String(event.timestamp ?? (/* @__PURE__ */ new Date()).toISOString())
|
|
2740
|
+
};
|
|
2741
|
+
} else if (type === "tool_use") {
|
|
2742
|
+
if (current) current.toolCount++;
|
|
2743
|
+
}
|
|
2744
|
+
}
|
|
2745
|
+
if (current) turns.push(current);
|
|
2746
|
+
return turns;
|
|
2747
|
+
}
|
|
2748
|
+
|
|
2693
2749
|
// src/artifacts/candidates.ts
|
|
2694
2750
|
import { execFileSync } from "child_process";
|
|
2695
2751
|
import fs4 from "fs";
|
|
@@ -2997,6 +3053,7 @@ async function handleRebuild(deps) {
|
|
|
2997
3053
|
config: deps.config,
|
|
2998
3054
|
index: deps.index,
|
|
2999
3055
|
vectorIndex: deps.vectorIndex ?? void 0,
|
|
3056
|
+
pipeline: deps.pipeline,
|
|
3000
3057
|
log: deps.log
|
|
3001
3058
|
},
|
|
3002
3059
|
deps.embeddingProvider,
|
|
@@ -3046,9 +3103,10 @@ async function handleDigest(deps, body) {
|
|
|
3046
3103
|
config: deps.config,
|
|
3047
3104
|
index: deps.index,
|
|
3048
3105
|
vectorIndex: deps.vectorIndex ?? void 0,
|
|
3106
|
+
pipeline: deps.pipeline,
|
|
3049
3107
|
log: deps.log
|
|
3050
3108
|
},
|
|
3051
|
-
deps.
|
|
3109
|
+
deps.digestLlmProvider,
|
|
3052
3110
|
options ?? void 0
|
|
3053
3111
|
).then((result) => {
|
|
3054
3112
|
if (result) {
|
|
@@ -3093,6 +3151,7 @@ async function handleCurate(deps, body, runCuration2) {
|
|
|
3093
3151
|
vectorIndex: deps.vectorIndex,
|
|
3094
3152
|
llmProvider: deps.llmProvider,
|
|
3095
3153
|
embeddingProvider: deps.embeddingProvider,
|
|
3154
|
+
pipeline: deps.pipeline,
|
|
3096
3155
|
log: deps.log
|
|
3097
3156
|
};
|
|
3098
3157
|
if (isDryRun) {
|
|
@@ -3144,6 +3203,7 @@ async function handleReprocess(deps, body) {
|
|
|
3144
3203
|
config: deps.config,
|
|
3145
3204
|
index: deps.index,
|
|
3146
3205
|
vectorIndex: deps.vectorIndex ?? void 0,
|
|
3206
|
+
pipeline: deps.pipeline,
|
|
3147
3207
|
log: deps.log
|
|
3148
3208
|
},
|
|
3149
3209
|
deps.llmProvider,
|
|
@@ -3250,47 +3310,1412 @@ function handleGetSessions(index) {
|
|
|
3250
3310
|
return { body: { sessions, dates } };
|
|
3251
3311
|
}
|
|
3252
3312
|
|
|
3253
|
-
// src/daemon/
|
|
3313
|
+
// src/daemon/api/pipeline.ts
|
|
3314
|
+
var RetryItemBody = external_exports.object({
|
|
3315
|
+
type: external_exports.string(),
|
|
3316
|
+
stage: external_exports.string()
|
|
3317
|
+
});
|
|
3318
|
+
function handlePipelineHealth(pipeline) {
|
|
3319
|
+
return async () => {
|
|
3320
|
+
const health = pipeline.health();
|
|
3321
|
+
return { body: health };
|
|
3322
|
+
};
|
|
3323
|
+
}
|
|
3324
|
+
function handlePipelineItems(pipeline) {
|
|
3325
|
+
return async (req) => {
|
|
3326
|
+
const { stage, status, type, limit, offset } = req.query;
|
|
3327
|
+
const parsedLimit = limit ? parseInt(limit, 10) : void 0;
|
|
3328
|
+
const parsedOffset = offset ? parseInt(offset, 10) : void 0;
|
|
3329
|
+
if (limit && (isNaN(parsedLimit) || parsedLimit < 0)) {
|
|
3330
|
+
return { status: 400, body: { error: "invalid_limit", message: "limit must be a non-negative integer" } };
|
|
3331
|
+
}
|
|
3332
|
+
if (offset && (isNaN(parsedOffset) || parsedOffset < 0)) {
|
|
3333
|
+
return { status: 400, body: { error: "invalid_offset", message: "offset must be a non-negative integer" } };
|
|
3334
|
+
}
|
|
3335
|
+
const result = pipeline.listItems({
|
|
3336
|
+
stage,
|
|
3337
|
+
status,
|
|
3338
|
+
type,
|
|
3339
|
+
limit: parsedLimit,
|
|
3340
|
+
offset: parsedOffset
|
|
3341
|
+
});
|
|
3342
|
+
return { body: result };
|
|
3343
|
+
};
|
|
3344
|
+
}
|
|
3345
|
+
function handlePipelineItemDetail(pipeline) {
|
|
3346
|
+
return async (req) => {
|
|
3347
|
+
const { id } = req.params;
|
|
3348
|
+
const { type } = req.query;
|
|
3349
|
+
if (!type) {
|
|
3350
|
+
return { status: 400, body: { error: "missing_type", message: 'query param "type" is required' } };
|
|
3351
|
+
}
|
|
3352
|
+
const stages = pipeline.getItemStatus(id, type);
|
|
3353
|
+
if (stages.length === 0) {
|
|
3354
|
+
return { status: 404, body: { error: "not_found", message: `No work item found: ${id} (${type})` } };
|
|
3355
|
+
}
|
|
3356
|
+
const history = pipeline.getTransitionHistory(id, type);
|
|
3357
|
+
return { body: { id, type, stages, history } };
|
|
3358
|
+
};
|
|
3359
|
+
}
|
|
3360
|
+
function handlePipelineCircuits(pipeline) {
|
|
3361
|
+
return async () => {
|
|
3362
|
+
const circuits = pipeline.listCircuits();
|
|
3363
|
+
return { body: circuits };
|
|
3364
|
+
};
|
|
3365
|
+
}
|
|
3366
|
+
function handlePipelineRetry(pipeline) {
|
|
3367
|
+
return async (req) => {
|
|
3368
|
+
const { id } = req.params;
|
|
3369
|
+
const parsed = RetryItemBody.safeParse(req.body);
|
|
3370
|
+
if (!parsed.success) {
|
|
3371
|
+
return { status: 400, body: { error: "validation_failed", issues: parsed.error.issues } };
|
|
3372
|
+
}
|
|
3373
|
+
const { type, stage } = parsed.data;
|
|
3374
|
+
const retried = pipeline.retryItem(id, type, stage);
|
|
3375
|
+
if (!retried) {
|
|
3376
|
+
return { status: 404, body: { error: "not_poisoned", message: `Item ${id} is not poisoned at stage ${stage}` } };
|
|
3377
|
+
}
|
|
3378
|
+
return { body: { retried: true, id, type, stage } };
|
|
3379
|
+
};
|
|
3380
|
+
}
|
|
3381
|
+
function handlePipelineSkip(pipeline) {
|
|
3382
|
+
return async (req) => {
|
|
3383
|
+
const { id } = req.params;
|
|
3384
|
+
const parsed = RetryItemBody.safeParse(req.body);
|
|
3385
|
+
if (!parsed.success) {
|
|
3386
|
+
return { status: 400, body: { error: "validation_failed", issues: parsed.error.issues } };
|
|
3387
|
+
}
|
|
3388
|
+
const { type, stage } = parsed.data;
|
|
3389
|
+
const skipped = pipeline.skipItem(id, type, stage);
|
|
3390
|
+
if (!skipped) {
|
|
3391
|
+
return { status: 404, body: { error: "not_skippable", message: `Item ${id} is not failed or poisoned at stage ${stage}` } };
|
|
3392
|
+
}
|
|
3393
|
+
return { body: { skipped: true, id, type, stage } };
|
|
3394
|
+
};
|
|
3395
|
+
}
|
|
3396
|
+
function handlePipelineRetryAll(pipeline) {
|
|
3397
|
+
return async () => {
|
|
3398
|
+
const count = pipeline.retryAllPoisoned();
|
|
3399
|
+
return { body: { retried: count } };
|
|
3400
|
+
};
|
|
3401
|
+
}
|
|
3402
|
+
function handlePipelineCircuitReset(pipeline) {
|
|
3403
|
+
return async (req) => {
|
|
3404
|
+
const { provider } = req.params;
|
|
3405
|
+
if (!PIPELINE_PROVIDER_ROLES.includes(provider)) {
|
|
3406
|
+
return {
|
|
3407
|
+
status: 400,
|
|
3408
|
+
body: {
|
|
3409
|
+
error: "unknown_provider",
|
|
3410
|
+
message: `Unknown provider role '${provider}'. Valid roles: ${PIPELINE_PROVIDER_ROLES.join(", ")}`
|
|
3411
|
+
}
|
|
3412
|
+
};
|
|
3413
|
+
}
|
|
3414
|
+
pipeline.resetCircuit(provider);
|
|
3415
|
+
const unblocked = pipeline.unblockItemsForCircuit(provider);
|
|
3416
|
+
return { body: { reset: true, provider, unblocked } };
|
|
3417
|
+
};
|
|
3418
|
+
}
|
|
3419
|
+
|
|
3420
|
+
// src/daemon/api/digest.ts
|
|
3254
3421
|
var import_yaml = __toESM(require_dist(), 1);
|
|
3255
3422
|
import fs6 from "fs";
|
|
3256
3423
|
import path9 from "path";
|
|
3257
|
-
function
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3424
|
+
function handleForceDigest(setForceDigest) {
|
|
3425
|
+
return async () => {
|
|
3426
|
+
setForceDigest();
|
|
3427
|
+
return {
|
|
3428
|
+
body: {
|
|
3429
|
+
status: "queued",
|
|
3430
|
+
message: "Digest will run on next pipeline tick (upstream must be clear)"
|
|
3431
|
+
}
|
|
3432
|
+
};
|
|
3433
|
+
};
|
|
3434
|
+
}
|
|
3435
|
+
function readExtractTimestamp(extractPath) {
|
|
3436
|
+
try {
|
|
3437
|
+
const content = fs6.readFileSync(extractPath, "utf-8");
|
|
3438
|
+
const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
|
|
3439
|
+
if (!fmMatch) return null;
|
|
3440
|
+
const fm = import_yaml.default.parse(fmMatch[1]);
|
|
3441
|
+
return typeof fm.generated === "string" ? fm.generated : null;
|
|
3442
|
+
} catch {
|
|
3443
|
+
return null;
|
|
3261
3444
|
}
|
|
3262
3445
|
}
|
|
3263
|
-
function
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3446
|
+
function handleDigestHealth(deps) {
|
|
3447
|
+
return async () => {
|
|
3448
|
+
const tracePath = path9.join(deps.vaultDir, "digest", "trace.jsonl");
|
|
3449
|
+
const lastCycle = readLastRecord(tracePath);
|
|
3450
|
+
let lastCycleTimestamp = lastCycle?.timestamp ?? null;
|
|
3451
|
+
const digestDir = path9.join(deps.vaultDir, "digest");
|
|
3452
|
+
try {
|
|
3453
|
+
for (const file of fs6.readdirSync(digestDir)) {
|
|
3454
|
+
if (!file.startsWith("extract-") || !file.endsWith(".md")) continue;
|
|
3455
|
+
const extractTs = readExtractTimestamp(path9.join(digestDir, file));
|
|
3456
|
+
if (extractTs && (!lastCycleTimestamp || extractTs > lastCycleTimestamp)) {
|
|
3457
|
+
lastCycleTimestamp = extractTs;
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
3460
|
+
} catch {
|
|
3461
|
+
}
|
|
3462
|
+
const substrateReady = deps.pipeline.newSubstrateSinceLastDigest();
|
|
3463
|
+
return {
|
|
3464
|
+
body: {
|
|
3465
|
+
last_cycle: lastCycleTimestamp ? {
|
|
3466
|
+
cycle_id: lastCycle?.cycleId ?? null,
|
|
3467
|
+
timestamp: lastCycleTimestamp,
|
|
3468
|
+
substrate_count: lastCycle ? Object.values(lastCycle.substrate).flat().length : null,
|
|
3469
|
+
tiers_generated: lastCycle?.tiersGenerated ?? null,
|
|
3470
|
+
duration_ms: lastCycle?.durationMs ?? null,
|
|
3471
|
+
model: lastCycle?.model ?? null
|
|
3472
|
+
} : null,
|
|
3473
|
+
substrate_ready: substrateReady,
|
|
3474
|
+
substrate_threshold: deps.minNotesForCycle,
|
|
3475
|
+
metabolism_state: deps.metabolismState(),
|
|
3476
|
+
digest_ready: deps.digestReady(),
|
|
3477
|
+
cycle_in_progress: deps.cycleInProgress()
|
|
3478
|
+
}
|
|
3479
|
+
};
|
|
3480
|
+
};
|
|
3481
|
+
}
|
|
3482
|
+
|
|
3483
|
+
// src/daemon/pipeline.ts
|
|
3484
|
+
import Database from "better-sqlite3";
|
|
3485
|
+
import fs7 from "fs";
|
|
3486
|
+
import path10 from "path";
|
|
3487
|
+
|
|
3488
|
+
// src/daemon/pipeline-classify.ts
|
|
3489
|
+
var WELL_KNOWN_API_HOSTS = ["api.anthropic.com", "api.openai.com"];
|
|
3490
|
+
var TRANSIENT_STATUS_PATTERNS = [" 429 ", " 500 ", " 503 "];
|
|
3491
|
+
var MODEL_NOT_FOUND_PATTERNS = [
|
|
3492
|
+
"model not found",
|
|
3493
|
+
"model not loaded",
|
|
3494
|
+
"no model loaded"
|
|
3495
|
+
];
|
|
3496
|
+
var RESOURCE_FAILURE_PATTERNS = [
|
|
3497
|
+
"model_load_failed",
|
|
3498
|
+
"insufficient system resources",
|
|
3499
|
+
"unsupported",
|
|
3500
|
+
"not compatible"
|
|
3501
|
+
];
|
|
3502
|
+
var TRANSIENT_MESSAGE_PATTERNS = [
|
|
3503
|
+
"socket hang up"
|
|
3504
|
+
];
|
|
3505
|
+
var CONFIG_ERROR_CODES = /* @__PURE__ */ new Set(["ECONNREFUSED"]);
|
|
3506
|
+
var TRANSIENT_ERROR_CODES = /* @__PURE__ */ new Set(["ETIMEDOUT", "ECONNRESET"]);
|
|
3507
|
+
function hasCode(err, code) {
|
|
3508
|
+
return err.code === code;
|
|
3509
|
+
}
|
|
3510
|
+
function hasAnyCode(err, codes) {
|
|
3511
|
+
const code = err.code;
|
|
3512
|
+
return code !== void 0 && codes.has(code);
|
|
3513
|
+
}
|
|
3514
|
+
function messageContains(err, patterns) {
|
|
3515
|
+
const msg = err.message.toLowerCase();
|
|
3516
|
+
return patterns.some((p) => msg.includes(p.toLowerCase()));
|
|
3517
|
+
}
|
|
3518
|
+
function statusMatch(err, patterns) {
|
|
3519
|
+
const msg = " " + err.message + " ";
|
|
3520
|
+
return patterns.some((p) => msg.includes(p));
|
|
3521
|
+
}
|
|
3522
|
+
function connectionRefusedAction(ctx) {
|
|
3523
|
+
const provider = ctx.providerName ?? "the provider";
|
|
3524
|
+
const url = ctx.baseUrl ? ` at ${ctx.baseUrl}` : "";
|
|
3525
|
+
return `${provider} is not reachable${url}. Check that ${provider} is running and the baseUrl is correct.`;
|
|
3526
|
+
}
|
|
3527
|
+
function modelNotFoundAction(ctx) {
|
|
3528
|
+
const model = ctx.modelName ? `'${ctx.modelName}'` : "the configured model";
|
|
3529
|
+
const provider = ctx.providerName ?? "the provider";
|
|
3530
|
+
return `Model ${model} was not found in ${provider}. Ensure the model is downloaded and loaded.`;
|
|
3531
|
+
}
|
|
3532
|
+
function resourceExhaustionAction(ctx) {
|
|
3533
|
+
const provider = ctx.providerName ?? "the provider";
|
|
3534
|
+
return `${provider} could not load the model due to insufficient resources. Try a smaller model or free up memory.`;
|
|
3535
|
+
}
|
|
3536
|
+
function authFailureAction(status, ctx) {
|
|
3537
|
+
const provider = ctx.providerName ?? "the provider";
|
|
3538
|
+
if (status === "401") {
|
|
3539
|
+
return `Authentication failed for ${provider}. Check that the API key is set and valid.`;
|
|
3540
|
+
}
|
|
3541
|
+
return `Access denied (403) for ${provider}. Check API key permissions and account status.`;
|
|
3542
|
+
}
|
|
3543
|
+
function dnsFailureAction(ctx) {
|
|
3544
|
+
const provider = ctx.providerName ?? "the provider";
|
|
3545
|
+
const host = ctx.configuredHost ?? "the configured host";
|
|
3546
|
+
return `Cannot resolve host '${host}' for ${provider}. Check the baseUrl hostname.`;
|
|
3547
|
+
}
|
|
3548
|
+
function classifyError(error, context) {
|
|
3549
|
+
const ctx = context ?? {};
|
|
3550
|
+
if (error instanceof SyntaxError) {
|
|
3551
|
+
return { type: "parse", suggestedAction: "LLM returned unparseable JSON. The response may be malformed or truncated." };
|
|
3552
|
+
}
|
|
3553
|
+
if (error.name === "ParseError") {
|
|
3554
|
+
return { type: "parse", suggestedAction: "LLM returned an empty or invalid response. Check model health and prompt." };
|
|
3555
|
+
}
|
|
3556
|
+
const msgLower = error.message.toLowerCase();
|
|
3557
|
+
if (msgLower.includes("empty content") || msgLower.includes("schema validation")) {
|
|
3558
|
+
return { type: "parse", suggestedAction: "LLM returned an empty or schema-invalid response. Check model health and prompt." };
|
|
3559
|
+
}
|
|
3560
|
+
if (msgLower.includes("only reasoning") || msgLower.includes("empty after strip") || msgLower.includes("observation extraction failed")) {
|
|
3561
|
+
return { type: "parse", suggestedAction: "LLM returned only reasoning tokens with no usable content. Check model reasoning settings." };
|
|
3562
|
+
}
|
|
3563
|
+
if (msgLower.includes("missing output") || msgLower.includes("no content in response") || msgLower.includes("summarization failed")) {
|
|
3564
|
+
return { type: "parse", suggestedAction: "LLM response is missing expected fields. Check provider health and model compatibility." };
|
|
3565
|
+
}
|
|
3566
|
+
if (hasCode(error, "ECONNREFUSED") || hasAnyCode(error, CONFIG_ERROR_CODES)) {
|
|
3567
|
+
return { type: "config", suggestedAction: connectionRefusedAction(ctx) };
|
|
3568
|
+
}
|
|
3569
|
+
if (messageContains(error, MODEL_NOT_FOUND_PATTERNS)) {
|
|
3570
|
+
return { type: "config", suggestedAction: modelNotFoundAction(ctx) };
|
|
3571
|
+
}
|
|
3572
|
+
if (messageContains(error, RESOURCE_FAILURE_PATTERNS)) {
|
|
3573
|
+
return { type: "config", suggestedAction: resourceExhaustionAction(ctx) };
|
|
3574
|
+
}
|
|
3575
|
+
if (statusMatch(error, [" 401 "])) {
|
|
3576
|
+
return { type: "config", suggestedAction: authFailureAction("401", ctx) };
|
|
3577
|
+
}
|
|
3578
|
+
if (statusMatch(error, [" 403 "])) {
|
|
3579
|
+
return { type: "config", suggestedAction: authFailureAction("403", ctx) };
|
|
3580
|
+
}
|
|
3581
|
+
if (/ 401$/.test(error.message) || error.message.endsWith(" 401")) {
|
|
3582
|
+
return { type: "config", suggestedAction: authFailureAction("401", ctx) };
|
|
3583
|
+
}
|
|
3584
|
+
if (/ 403$/.test(error.message) || error.message.endsWith(" 403")) {
|
|
3585
|
+
return { type: "config", suggestedAction: authFailureAction("403", ctx) };
|
|
3586
|
+
}
|
|
3587
|
+
if (hasCode(error, "ENOTFOUND")) {
|
|
3588
|
+
if (ctx.configuredHost && error.message.includes(ctx.configuredHost)) {
|
|
3589
|
+
return { type: "config", suggestedAction: dnsFailureAction(ctx) };
|
|
3590
|
+
}
|
|
3591
|
+
if (WELL_KNOWN_API_HOSTS.some((host) => error.message.includes(host))) {
|
|
3592
|
+
return { type: "transient" };
|
|
3593
|
+
}
|
|
3594
|
+
return { type: "config", suggestedAction: dnsFailureAction(ctx) };
|
|
3595
|
+
}
|
|
3596
|
+
if (error.name === "AbortError") {
|
|
3597
|
+
return { type: "transient" };
|
|
3598
|
+
}
|
|
3599
|
+
if (hasAnyCode(error, TRANSIENT_ERROR_CODES)) {
|
|
3600
|
+
return { type: "transient" };
|
|
3601
|
+
}
|
|
3602
|
+
if (statusMatch(error, TRANSIENT_STATUS_PATTERNS)) {
|
|
3603
|
+
return { type: "transient" };
|
|
3604
|
+
}
|
|
3605
|
+
if (messageContains(error, TRANSIENT_MESSAGE_PATTERNS)) {
|
|
3606
|
+
return { type: "transient" };
|
|
3607
|
+
}
|
|
3608
|
+
return { type: "transient" };
|
|
3609
|
+
}
|
|
3610
|
+
|
|
3611
|
+
// src/daemon/pipeline.ts
|
|
3612
|
+
var PIPELINE_DB_FILENAME = "pipeline.db";
|
|
3613
|
+
var SCHEMA_SQL = `
|
|
3614
|
+
-- work_items: every piece of content in the pipeline
|
|
3615
|
+
CREATE TABLE IF NOT EXISTS work_items (
|
|
3616
|
+
id TEXT NOT NULL,
|
|
3617
|
+
item_type TEXT NOT NULL,
|
|
3618
|
+
source_path TEXT,
|
|
3619
|
+
created_at TEXT NOT NULL,
|
|
3620
|
+
updated_at TEXT NOT NULL,
|
|
3621
|
+
PRIMARY KEY (id, item_type)
|
|
3622
|
+
);
|
|
3623
|
+
|
|
3624
|
+
-- stage_transitions: append-only audit trail
|
|
3625
|
+
CREATE TABLE IF NOT EXISTS stage_transitions (
|
|
3626
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
3627
|
+
work_item_id TEXT NOT NULL,
|
|
3628
|
+
item_type TEXT NOT NULL,
|
|
3629
|
+
stage TEXT NOT NULL,
|
|
3630
|
+
status TEXT NOT NULL,
|
|
3631
|
+
attempt INTEGER DEFAULT 1,
|
|
3632
|
+
error_type TEXT,
|
|
3633
|
+
error_message TEXT,
|
|
3634
|
+
started_at TEXT,
|
|
3635
|
+
completed_at TEXT,
|
|
3636
|
+
output TEXT,
|
|
3637
|
+
created_at TEXT NOT NULL,
|
|
3638
|
+
FOREIGN KEY (work_item_id, item_type) REFERENCES work_items(id, item_type)
|
|
3639
|
+
);
|
|
3640
|
+
|
|
3641
|
+
-- stage_history: compacted transitions older than retention window
|
|
3642
|
+
CREATE TABLE IF NOT EXISTS stage_history (
|
|
3643
|
+
work_item_id TEXT NOT NULL,
|
|
3644
|
+
item_type TEXT NOT NULL,
|
|
3645
|
+
stage TEXT NOT NULL,
|
|
3646
|
+
total_attempts INTEGER,
|
|
3647
|
+
final_status TEXT NOT NULL,
|
|
3648
|
+
first_attempt TEXT NOT NULL,
|
|
3649
|
+
last_attempt TEXT NOT NULL,
|
|
3650
|
+
last_error TEXT,
|
|
3651
|
+
error_types TEXT,
|
|
3652
|
+
last_output TEXT,
|
|
3653
|
+
PRIMARY KEY (work_item_id, item_type, stage)
|
|
3654
|
+
);
|
|
3655
|
+
|
|
3656
|
+
-- circuit_breakers: per-provider-role state
|
|
3657
|
+
CREATE TABLE IF NOT EXISTS circuit_breakers (
|
|
3658
|
+
provider_role TEXT PRIMARY KEY,
|
|
3659
|
+
state TEXT NOT NULL DEFAULT 'closed',
|
|
3660
|
+
failure_count INTEGER DEFAULT 0,
|
|
3661
|
+
last_failure TEXT,
|
|
3662
|
+
last_error TEXT,
|
|
3663
|
+
opens_at TEXT,
|
|
3664
|
+
updated_at TEXT NOT NULL
|
|
3665
|
+
);
|
|
3666
|
+
`;
|
|
3667
|
+
var INDEXES_SQL = `
|
|
3668
|
+
CREATE INDEX IF NOT EXISTS idx_transitions_item_stage
|
|
3669
|
+
ON stage_transitions(work_item_id, item_type, stage);
|
|
3670
|
+
CREATE INDEX IF NOT EXISTS idx_transitions_status
|
|
3671
|
+
ON stage_transitions(status);
|
|
3672
|
+
CREATE INDEX IF NOT EXISTS idx_items_type
|
|
3673
|
+
ON work_items(item_type);
|
|
3674
|
+
`;
|
|
3675
|
+
var VIEW_SQL = `
|
|
3676
|
+
CREATE VIEW IF NOT EXISTS pipeline_status AS
|
|
3677
|
+
WITH ranked AS (
|
|
3678
|
+
SELECT st.*,
|
|
3679
|
+
ROW_NUMBER() OVER (
|
|
3680
|
+
PARTITION BY st.work_item_id, st.item_type, st.stage
|
|
3681
|
+
ORDER BY st.id DESC
|
|
3682
|
+
) AS rn
|
|
3683
|
+
FROM stage_transitions st
|
|
3684
|
+
)
|
|
3685
|
+
SELECT
|
|
3686
|
+
wi.id, wi.item_type, wi.source_path,
|
|
3687
|
+
r.stage, r.status, r.attempt,
|
|
3688
|
+
r.error_type, r.error_message,
|
|
3689
|
+
r.started_at, r.completed_at,
|
|
3690
|
+
r.output
|
|
3691
|
+
FROM work_items wi
|
|
3692
|
+
JOIN ranked r ON r.work_item_id = wi.id AND r.item_type = wi.item_type
|
|
3693
|
+
WHERE r.rn = 1;
|
|
3694
|
+
`;
|
|
3695
|
+
var HEALTH_STAGE_STATUS_SQL = `
|
|
3696
|
+
SELECT stage, status, COUNT(*) as count
|
|
3697
|
+
FROM pipeline_status
|
|
3698
|
+
GROUP BY stage, status
|
|
3699
|
+
`;
|
|
3700
|
+
var HEALTH_CIRCUITS_SQL = `
|
|
3701
|
+
SELECT provider_role, state, failure_count, last_error
|
|
3702
|
+
FROM circuit_breakers
|
|
3703
|
+
`;
|
|
3704
|
+
var PipelineManager = class {
|
|
3705
|
+
db;
|
|
3706
|
+
config;
|
|
3707
|
+
constructor(vaultDir, config) {
|
|
3708
|
+
const dbPath = path10.join(vaultDir, PIPELINE_DB_FILENAME);
|
|
3709
|
+
this.db = new Database(dbPath);
|
|
3710
|
+
this.db.pragma("journal_mode = WAL");
|
|
3711
|
+
this.db.pragma("foreign_keys = ON");
|
|
3712
|
+
this.config = config;
|
|
3713
|
+
this.init();
|
|
3714
|
+
}
|
|
3715
|
+
init() {
|
|
3716
|
+
this.db.exec("DROP VIEW IF EXISTS pipeline_status");
|
|
3717
|
+
this.db.exec(SCHEMA_SQL);
|
|
3718
|
+
try {
|
|
3719
|
+
this.db.exec("ALTER TABLE stage_transitions ADD COLUMN output TEXT");
|
|
3720
|
+
} catch {
|
|
3721
|
+
}
|
|
3722
|
+
try {
|
|
3723
|
+
this.db.exec("ALTER TABLE stage_history ADD COLUMN last_output TEXT");
|
|
3724
|
+
} catch {
|
|
3725
|
+
}
|
|
3726
|
+
this.db.exec(INDEXES_SQL);
|
|
3727
|
+
this.db.exec(VIEW_SQL);
|
|
3728
|
+
}
|
|
3729
|
+
/** Expose the underlying database for direct queries (used in tests and by higher-level methods). */
|
|
3730
|
+
getDb() {
|
|
3731
|
+
return this.db;
|
|
3732
|
+
}
|
|
3733
|
+
/** Read a PRAGMA value (used in tests to verify WAL mode and foreign keys). */
|
|
3734
|
+
getPragma(name) {
|
|
3735
|
+
return this.db.pragma(name, { simple: true });
|
|
3736
|
+
}
|
|
3737
|
+
/** Quick check whether the pipeline has any registered work items. */
|
|
3738
|
+
isEmpty() {
|
|
3739
|
+
const row = this.db.prepare("SELECT 1 FROM work_items LIMIT 1").get();
|
|
3740
|
+
return !row;
|
|
3741
|
+
}
|
|
3742
|
+
/** Aggregate pipeline health: stage/status counts, circuit states, totals. */
|
|
3743
|
+
health() {
|
|
3744
|
+
const stageRows = this.db.prepare(HEALTH_STAGE_STATUS_SQL).all();
|
|
3745
|
+
const stages = {};
|
|
3746
|
+
const totals = {
|
|
3747
|
+
pending: 0,
|
|
3748
|
+
processing: 0,
|
|
3749
|
+
failed: 0,
|
|
3750
|
+
blocked: 0,
|
|
3751
|
+
poisoned: 0,
|
|
3752
|
+
succeeded: 0
|
|
3753
|
+
};
|
|
3754
|
+
for (const row of stageRows) {
|
|
3755
|
+
if (!stages[row.stage]) {
|
|
3756
|
+
stages[row.stage] = {};
|
|
3757
|
+
}
|
|
3758
|
+
stages[row.stage][row.status] = row.count;
|
|
3759
|
+
if (row.status in totals) {
|
|
3760
|
+
totals[row.status] += row.count;
|
|
3761
|
+
}
|
|
3762
|
+
}
|
|
3763
|
+
const circuitRows = this.db.prepare(HEALTH_CIRCUITS_SQL).all();
|
|
3764
|
+
const circuits = circuitRows.map((r) => ({
|
|
3765
|
+
provider_role: r.provider_role,
|
|
3766
|
+
state: r.state,
|
|
3767
|
+
failure_count: r.failure_count,
|
|
3768
|
+
last_error: r.last_error
|
|
3769
|
+
}));
|
|
3770
|
+
return { stages, circuits, totals };
|
|
3771
|
+
}
|
|
3772
|
+
// -------------------------------------------------------------------------
|
|
3773
|
+
// Work item registration
|
|
3774
|
+
// -------------------------------------------------------------------------
|
|
3775
|
+
/**
|
|
3776
|
+
* Register a work item in the pipeline. Creates the work_items row and
|
|
3777
|
+
* initial stage_transitions for all applicable stages.
|
|
3778
|
+
*
|
|
3779
|
+
* Uses INSERT OR IGNORE for work_items (idempotency — re-registering is a no-op).
|
|
3780
|
+
* Checks if transitions already exist before inserting.
|
|
3781
|
+
*/
|
|
3782
|
+
register(itemId, itemType, sourcePath) {
|
|
3783
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3784
|
+
this.db.prepare(
|
|
3785
|
+
"INSERT OR IGNORE INTO work_items (id, item_type, source_path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)"
|
|
3786
|
+
).run(itemId, itemType, sourcePath ?? null, now, now);
|
|
3787
|
+
const existingCount = this.db.prepare(
|
|
3788
|
+
"SELECT COUNT(*) as cnt FROM stage_transitions WHERE work_item_id = ? AND item_type = ?"
|
|
3789
|
+
).get(itemId, itemType);
|
|
3790
|
+
if (existingCount.cnt > 0) {
|
|
3791
|
+
return;
|
|
3792
|
+
}
|
|
3793
|
+
const applicableStages = ITEM_STAGE_MAP[itemType] ?? [];
|
|
3794
|
+
const insertStmt = this.db.prepare(
|
|
3795
|
+
"INSERT INTO stage_transitions (work_item_id, item_type, stage, status, attempt, created_at) VALUES (?, ?, ?, ?, ?, ?)"
|
|
3796
|
+
);
|
|
3797
|
+
const insertAll = this.db.transaction(() => {
|
|
3798
|
+
for (const stage of PIPELINE_STAGES) {
|
|
3799
|
+
const status = applicableStages.includes(stage) ? "pending" : "skipped";
|
|
3800
|
+
insertStmt.run(itemId, itemType, stage, status, 1, now);
|
|
3801
|
+
}
|
|
3802
|
+
});
|
|
3803
|
+
insertAll();
|
|
3804
|
+
}
|
|
3805
|
+
// -------------------------------------------------------------------------
|
|
3806
|
+
// Stage transitions
|
|
3807
|
+
// -------------------------------------------------------------------------
|
|
3808
|
+
/**
|
|
3809
|
+
* Record a stage transition. Append-only — never updates existing rows.
|
|
3810
|
+
*
|
|
3811
|
+
* When status is 'failed': checks retry limits and may auto-poison.
|
|
3812
|
+
* When error_type is 'config': blocks all downstream stages.
|
|
3813
|
+
*/
|
|
3814
|
+
advance(itemId, itemType, stage, status, error, output) {
|
|
3815
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3816
|
+
const priorCount = this.db.prepare(
|
|
3817
|
+
`SELECT COUNT(*) as cnt FROM stage_transitions
|
|
3818
|
+
WHERE work_item_id = ? AND item_type = ? AND stage = ?
|
|
3819
|
+
AND status IN ('failed', 'processing')`
|
|
3820
|
+
).get(itemId, itemType, stage);
|
|
3821
|
+
const attempt = status === "failed" || status === "processing" ? Math.ceil((priorCount.cnt + 1) / 2) || 1 : priorCount.cnt > 0 ? Math.ceil(priorCount.cnt / 2) || 1 : 1;
|
|
3822
|
+
let resolvedStatus = status;
|
|
3823
|
+
if (status === "failed" && error?.errorType) {
|
|
3824
|
+
const maxRetries = error.errorType === "transient" ? this.config.retry.transient_max : PIPELINE_PARSE_MAX_RETRIES;
|
|
3825
|
+
const failedCount = this.db.prepare(
|
|
3826
|
+
`SELECT COUNT(*) as cnt FROM stage_transitions
|
|
3827
|
+
WHERE work_item_id = ? AND item_type = ? AND stage = ? AND status = 'failed'`
|
|
3828
|
+
).get(itemId, itemType, stage);
|
|
3829
|
+
if (failedCount.cnt >= maxRetries) {
|
|
3830
|
+
resolvedStatus = "poisoned";
|
|
3831
|
+
}
|
|
3832
|
+
}
|
|
3833
|
+
const startedAt = status === "processing" ? now : null;
|
|
3834
|
+
const completedAt = ["succeeded", "failed", "poisoned", "blocked", "skipped"].includes(resolvedStatus) ? now : null;
|
|
3835
|
+
const outputJson = status === "succeeded" && output ? JSON.stringify(output) : null;
|
|
3836
|
+
this.db.prepare(
|
|
3837
|
+
`INSERT INTO stage_transitions
|
|
3838
|
+
(work_item_id, item_type, stage, status, attempt, error_type, error_message, started_at, completed_at, output, created_at)
|
|
3839
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
3840
|
+
).run(
|
|
3841
|
+
itemId,
|
|
3842
|
+
itemType,
|
|
3843
|
+
stage,
|
|
3844
|
+
resolvedStatus,
|
|
3845
|
+
attempt,
|
|
3846
|
+
error?.errorType ?? null,
|
|
3847
|
+
error?.errorMessage ?? null,
|
|
3848
|
+
startedAt,
|
|
3849
|
+
completedAt,
|
|
3850
|
+
outputJson,
|
|
3851
|
+
now
|
|
3273
3852
|
);
|
|
3274
|
-
|
|
3853
|
+
this.db.prepare("UPDATE work_items SET updated_at = ? WHERE id = ? AND item_type = ?").run(now, itemId, itemType);
|
|
3854
|
+
if (status === "failed" && error?.errorType === "config") {
|
|
3855
|
+
const stageIdx = PIPELINE_STAGES.indexOf(stage);
|
|
3856
|
+
if (stageIdx >= 0) {
|
|
3857
|
+
const downstreamStages = PIPELINE_STAGES.slice(stageIdx + 1);
|
|
3858
|
+
for (const downstream of downstreamStages) {
|
|
3859
|
+
const currentStatus = this.db.prepare(
|
|
3860
|
+
`SELECT status FROM stage_transitions
|
|
3861
|
+
WHERE work_item_id = ? AND item_type = ? AND stage = ?
|
|
3862
|
+
ORDER BY id DESC LIMIT 1`
|
|
3863
|
+
).get(itemId, itemType, downstream);
|
|
3864
|
+
if (currentStatus && currentStatus.status === "pending") {
|
|
3865
|
+
this.db.prepare(
|
|
3866
|
+
`INSERT INTO stage_transitions
|
|
3867
|
+
(work_item_id, item_type, stage, status, attempt, error_type, error_message, started_at, completed_at, created_at)
|
|
3868
|
+
VALUES (?, ?, ?, 'blocked', 1, 'config', ?, NULL, ?, ?)`
|
|
3869
|
+
).run(
|
|
3870
|
+
itemId,
|
|
3871
|
+
itemType,
|
|
3872
|
+
downstream,
|
|
3873
|
+
`blocked by ${stage} config failure`,
|
|
3874
|
+
now,
|
|
3875
|
+
now
|
|
3876
|
+
);
|
|
3877
|
+
}
|
|
3878
|
+
}
|
|
3879
|
+
}
|
|
3880
|
+
}
|
|
3275
3881
|
}
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3882
|
+
// -------------------------------------------------------------------------
|
|
3883
|
+
// Status queries
|
|
3884
|
+
// -------------------------------------------------------------------------
|
|
3885
|
+
/**
|
|
3886
|
+
* Get current status for all stages of a work item.
|
|
3887
|
+
* Queries the pipeline_status view filtered by work_item_id and item_type.
|
|
3888
|
+
*/
|
|
3889
|
+
getItemStatus(itemId, itemType) {
|
|
3890
|
+
return this.db.prepare(
|
|
3891
|
+
`SELECT stage, status, attempt, error_type, error_message, started_at, completed_at, output
|
|
3892
|
+
FROM pipeline_status
|
|
3893
|
+
WHERE id = ? AND item_type = ?`
|
|
3894
|
+
).all(itemId, itemType);
|
|
3895
|
+
}
|
|
3896
|
+
// -------------------------------------------------------------------------
|
|
3897
|
+
// Batch queries
|
|
3898
|
+
// -------------------------------------------------------------------------
|
|
3899
|
+
/**
|
|
3900
|
+
* Get pending work items ready for processing at a given stage.
|
|
3901
|
+
*
|
|
3902
|
+
* Requirements:
|
|
3903
|
+
* - Only items where the requested stage is 'pending'
|
|
3904
|
+
* - Only items whose PREVIOUS stage (in PIPELINE_STAGES order) is 'succeeded' or 'skipped'
|
|
3905
|
+
* - Exclude items in backoff window
|
|
3906
|
+
* - Ordered by work_items.created_at ASC (oldest first)
|
|
3907
|
+
*/
|
|
3908
|
+
nextBatch(stage, limit) {
|
|
3909
|
+
const stageIdx = PIPELINE_STAGES.indexOf(stage);
|
|
3910
|
+
const prevStage = stageIdx > 0 ? PIPELINE_STAGES[stageIdx - 1] : null;
|
|
3911
|
+
const upstreamCheck = prevStage ? `AND EXISTS (
|
|
3912
|
+
SELECT 1 FROM pipeline_status ps2
|
|
3913
|
+
WHERE ps2.id = ps.id AND ps2.item_type = ps.item_type
|
|
3914
|
+
AND ps2.stage = ? AND ps2.status IN ('succeeded', 'skipped')
|
|
3915
|
+
)` : "";
|
|
3916
|
+
const sql = `
|
|
3917
|
+
SELECT ps.id, ps.item_type, ps.source_path, wi.created_at
|
|
3918
|
+
FROM pipeline_status ps
|
|
3919
|
+
JOIN work_items wi ON wi.id = ps.id AND wi.item_type = ps.item_type
|
|
3920
|
+
WHERE ps.stage = ? AND ps.status IN ('pending', 'failed')
|
|
3921
|
+
${upstreamCheck}
|
|
3922
|
+
AND NOT EXISTS (
|
|
3923
|
+
SELECT 1 FROM stage_transitions st2
|
|
3924
|
+
WHERE st2.work_item_id = ps.id AND st2.item_type = ps.item_type
|
|
3925
|
+
AND st2.stage = ? AND st2.status = 'failed'
|
|
3926
|
+
AND st2.completed_at IS NOT NULL
|
|
3927
|
+
AND (julianday('now') - julianday(st2.completed_at)) * 86400000 <
|
|
3928
|
+
? * POWER(?, (
|
|
3929
|
+
SELECT COUNT(*) FROM stage_transitions st3
|
|
3930
|
+
WHERE st3.work_item_id = ps.id AND st3.item_type = ps.item_type
|
|
3931
|
+
AND st3.stage = ? AND st3.status = 'failed'
|
|
3932
|
+
) - 1)
|
|
3933
|
+
)
|
|
3934
|
+
ORDER BY wi.created_at ASC
|
|
3935
|
+
LIMIT ?`;
|
|
3936
|
+
const params = [stage];
|
|
3937
|
+
if (prevStage) {
|
|
3938
|
+
params.push(prevStage);
|
|
3939
|
+
}
|
|
3940
|
+
params.push(stage, this.config.retry.backoff_base_seconds * 1e3, PIPELINE_BACKOFF_MULTIPLIER, stage, limit);
|
|
3941
|
+
return this.db.prepare(sql).all(...params);
|
|
3942
|
+
}
|
|
3943
|
+
// -------------------------------------------------------------------------
|
|
3944
|
+
// Circuit breakers
|
|
3945
|
+
// -------------------------------------------------------------------------
|
|
3946
|
+
/**
|
|
3947
|
+
* Get current state of a circuit breaker for the given provider role.
|
|
3948
|
+
* Returns the persisted row, or a synthetic default (closed, 0 failures)
|
|
3949
|
+
* if no row exists yet.
|
|
3950
|
+
*/
|
|
3951
|
+
circuitState(providerRole) {
|
|
3952
|
+
const row = this.db.prepare("SELECT * FROM circuit_breakers WHERE provider_role = ?").get(providerRole);
|
|
3953
|
+
if (row) {
|
|
3954
|
+
return row;
|
|
3955
|
+
}
|
|
3956
|
+
return {
|
|
3957
|
+
provider_role: providerRole,
|
|
3958
|
+
state: "closed",
|
|
3959
|
+
failure_count: 0,
|
|
3960
|
+
last_failure: null,
|
|
3961
|
+
last_error: null,
|
|
3962
|
+
opens_at: null,
|
|
3963
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
3284
3964
|
};
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3965
|
+
}
|
|
3966
|
+
/**
|
|
3967
|
+
* Record a failure against a circuit breaker. Increments failure_count and
|
|
3968
|
+
* updates last_error / last_failure. If failure_count reaches the configured
|
|
3969
|
+
* failure_threshold, sets state to 'open' and calculates
|
|
3970
|
+
* opens_at = now + configured cooldown_seconds.
|
|
3971
|
+
*/
|
|
3972
|
+
tripCircuit(providerRole, errorMessage) {
|
|
3973
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3974
|
+
const current = this.circuitState(providerRole);
|
|
3975
|
+
const newFailureCount = current.failure_count + 1;
|
|
3976
|
+
const shouldOpen = newFailureCount >= this.config.circuit_breaker.failure_threshold;
|
|
3977
|
+
const newState = shouldOpen ? "open" : "closed";
|
|
3978
|
+
const opensAt = shouldOpen ? new Date(Date.now() + this.config.circuit_breaker.cooldown_seconds * 1e3).toISOString() : null;
|
|
3979
|
+
this.db.prepare(
|
|
3980
|
+
`INSERT INTO circuit_breakers
|
|
3981
|
+
(provider_role, state, failure_count, last_failure, last_error, opens_at, updated_at)
|
|
3982
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
3983
|
+
ON CONFLICT(provider_role) DO UPDATE SET
|
|
3984
|
+
state = excluded.state,
|
|
3985
|
+
failure_count = excluded.failure_count,
|
|
3986
|
+
last_failure = excluded.last_failure,
|
|
3987
|
+
last_error = excluded.last_error,
|
|
3988
|
+
opens_at = excluded.opens_at,
|
|
3989
|
+
updated_at = excluded.updated_at`
|
|
3990
|
+
).run(providerRole, newState, newFailureCount, now, errorMessage, opensAt, now);
|
|
3991
|
+
}
|
|
3992
|
+
/**
|
|
3993
|
+
* Re-open a circuit after a failed half-open probe with doubled cooldown.
|
|
3994
|
+
*
|
|
3995
|
+
* When a half-open probe fails, the circuit should re-open with a cooldown
|
|
3996
|
+
* of `previousCooldown * 2`, capped at the configured max_cooldown_seconds.
|
|
3997
|
+
* This implements exponential backoff for repeated probe failures.
|
|
3998
|
+
*/
|
|
3999
|
+
reopenCircuit(providerRole, errorMessage) {
|
|
4000
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4001
|
+
const current = this.circuitState(providerRole);
|
|
4002
|
+
let previousCooldown = this.config.circuit_breaker.cooldown_seconds * 1e3;
|
|
4003
|
+
if (current.last_failure && current.opens_at) {
|
|
4004
|
+
const lastFailureMs = new Date(current.last_failure).getTime();
|
|
4005
|
+
const opensAtMs = new Date(current.opens_at).getTime();
|
|
4006
|
+
const storedCooldown = opensAtMs - lastFailureMs;
|
|
4007
|
+
if (storedCooldown > 0) {
|
|
4008
|
+
previousCooldown = storedCooldown;
|
|
4009
|
+
}
|
|
4010
|
+
}
|
|
4011
|
+
const doubledCooldown = Math.min(
|
|
4012
|
+
previousCooldown * 2,
|
|
4013
|
+
this.config.circuit_breaker.max_cooldown_seconds * 1e3
|
|
4014
|
+
);
|
|
4015
|
+
const opensAt = new Date(Date.now() + doubledCooldown).toISOString();
|
|
4016
|
+
this.db.prepare(
|
|
4017
|
+
`INSERT INTO circuit_breakers
|
|
4018
|
+
(provider_role, state, failure_count, last_failure, last_error, opens_at, updated_at)
|
|
4019
|
+
VALUES (?, 'open', ?, ?, ?, ?, ?)
|
|
4020
|
+
ON CONFLICT(provider_role) DO UPDATE SET
|
|
4021
|
+
state = 'open',
|
|
4022
|
+
failure_count = excluded.failure_count,
|
|
4023
|
+
last_failure = excluded.last_failure,
|
|
4024
|
+
last_error = excluded.last_error,
|
|
4025
|
+
opens_at = excluded.opens_at,
|
|
4026
|
+
updated_at = excluded.updated_at`
|
|
4027
|
+
).run(providerRole, current.failure_count + 1, now, errorMessage, opensAt, now);
|
|
4028
|
+
}
|
|
4029
|
+
/**
|
|
4030
|
+
* Manually reset a circuit breaker to closed state.
|
|
4031
|
+
* Sets state='closed', failure_count=0, clears opens_at.
|
|
4032
|
+
*/
|
|
4033
|
+
resetCircuit(providerRole) {
|
|
4034
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4035
|
+
this.db.prepare(
|
|
4036
|
+
`INSERT INTO circuit_breakers
|
|
4037
|
+
(provider_role, state, failure_count, last_failure, last_error, opens_at, updated_at)
|
|
4038
|
+
VALUES (?, 'closed', 0, NULL, NULL, NULL, ?)
|
|
4039
|
+
ON CONFLICT(provider_role) DO UPDATE SET
|
|
4040
|
+
state = 'closed',
|
|
4041
|
+
failure_count = 0,
|
|
4042
|
+
opens_at = NULL,
|
|
4043
|
+
updated_at = excluded.updated_at`
|
|
4044
|
+
).run(providerRole, now);
|
|
4045
|
+
}
|
|
4046
|
+
/**
|
|
4047
|
+
* Check if an open circuit's cooldown has expired and is ready for a
|
|
4048
|
+
* half-open probe. If state is 'open' and current time >= opens_at,
|
|
4049
|
+
* transitions state to 'half-open' and returns true. Otherwise returns false.
|
|
4050
|
+
*/
|
|
4051
|
+
probeCircuit(providerRole) {
|
|
4052
|
+
const current = this.circuitState(providerRole);
|
|
4053
|
+
if (current.state !== "open") {
|
|
4054
|
+
return false;
|
|
4055
|
+
}
|
|
4056
|
+
if (!current.opens_at) {
|
|
4057
|
+
return false;
|
|
4058
|
+
}
|
|
4059
|
+
const now = Date.now();
|
|
4060
|
+
const opensAt = new Date(current.opens_at).getTime();
|
|
4061
|
+
if (now < opensAt) {
|
|
4062
|
+
return false;
|
|
4063
|
+
}
|
|
4064
|
+
this.db.prepare(
|
|
4065
|
+
`UPDATE circuit_breakers
|
|
4066
|
+
SET state = 'half-open', updated_at = ?
|
|
4067
|
+
WHERE provider_role = ?`
|
|
4068
|
+
).run((/* @__PURE__ */ new Date()).toISOString(), providerRole);
|
|
4069
|
+
return true;
|
|
4070
|
+
}
|
|
4071
|
+
/**
|
|
4072
|
+
* When a circuit opens, find all stages that use this provider role and
|
|
4073
|
+
* insert new 'blocked' transitions for all items that currently have
|
|
4074
|
+
* 'pending' status at those stages.
|
|
4075
|
+
*
|
|
4076
|
+
* Returns the count of blocked items.
|
|
4077
|
+
*/
|
|
4078
|
+
blockItemsForCircuit(providerRole) {
|
|
4079
|
+
return this._transitionItemsForCircuit(providerRole, "pending", "blocked");
|
|
4080
|
+
}
|
|
4081
|
+
/**
|
|
4082
|
+
* When a circuit closes, find all stages that use this provider role and
|
|
4083
|
+
* insert new 'pending' transitions for all items that currently have
|
|
4084
|
+
* 'blocked' status at those stages.
|
|
4085
|
+
*
|
|
4086
|
+
* Returns the count of unblocked items.
|
|
4087
|
+
*/
|
|
4088
|
+
unblockItemsForCircuit(providerRole) {
|
|
4089
|
+
return this._transitionItemsForCircuit(providerRole, "blocked", "pending");
|
|
4090
|
+
}
|
|
4091
|
+
/**
|
|
4092
|
+
* Shared implementation for blockItemsForCircuit / unblockItemsForCircuit.
|
|
4093
|
+
* Finds all stages mapped to providerRole, selects items at fromStatus,
|
|
4094
|
+
* and inserts a new transition at toStatus.
|
|
4095
|
+
*/
|
|
4096
|
+
_transitionItemsForCircuit(providerRole, fromStatus, toStatus) {
|
|
4097
|
+
const affectedStages = Object.keys(STAGE_PROVIDER_MAP).filter((stage) => STAGE_PROVIDER_MAP[stage] === providerRole);
|
|
4098
|
+
if (affectedStages.length === 0) {
|
|
4099
|
+
return 0;
|
|
4100
|
+
}
|
|
4101
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4102
|
+
let transitionedCount = 0;
|
|
4103
|
+
const isBlocking = toStatus === "blocked";
|
|
4104
|
+
const doTransition = this.db.transaction(() => {
|
|
4105
|
+
for (const stage of affectedStages) {
|
|
4106
|
+
const items = this.db.prepare(
|
|
4107
|
+
`SELECT id, item_type FROM pipeline_status
|
|
4108
|
+
WHERE stage = ? AND status = ?`
|
|
4109
|
+
).all(stage, fromStatus);
|
|
4110
|
+
for (const item of items) {
|
|
4111
|
+
if (isBlocking) {
|
|
4112
|
+
this.db.prepare(
|
|
4113
|
+
`INSERT INTO stage_transitions
|
|
4114
|
+
(work_item_id, item_type, stage, status, attempt, error_type, error_message, started_at, completed_at, created_at)
|
|
4115
|
+
VALUES (?, ?, ?, 'blocked', 1, 'config', ?, NULL, ?, ?)`
|
|
4116
|
+
).run(item.id, item.item_type, stage, `circuit open: ${providerRole}`, now, now);
|
|
4117
|
+
} else {
|
|
4118
|
+
this.db.prepare(
|
|
4119
|
+
`INSERT INTO stage_transitions
|
|
4120
|
+
(work_item_id, item_type, stage, status, attempt, error_type, error_message, started_at, completed_at, created_at)
|
|
4121
|
+
VALUES (?, ?, ?, 'pending', 1, NULL, NULL, NULL, NULL, ?)`
|
|
4122
|
+
).run(item.id, item.item_type, stage, now);
|
|
4123
|
+
}
|
|
4124
|
+
transitionedCount++;
|
|
4125
|
+
}
|
|
4126
|
+
}
|
|
4127
|
+
});
|
|
4128
|
+
doTransition();
|
|
4129
|
+
return transitionedCount;
|
|
4130
|
+
}
|
|
4131
|
+
// -------------------------------------------------------------------------
|
|
4132
|
+
// Compaction
|
|
4133
|
+
// -------------------------------------------------------------------------
|
|
4134
|
+
/**
|
|
4135
|
+
* Compact stage_transitions older than retentionDays into stage_history rows.
|
|
4136
|
+
*
|
|
4137
|
+
* For each (work_item_id, item_type, stage) group whose transitions are older
|
|
4138
|
+
* than the cutoff, inserts or replaces a stage_history row aggregating those
|
|
4139
|
+
* transitions, then deletes the original rows.
|
|
4140
|
+
*
|
|
4141
|
+
* Returns `{ compacted, deleted }`: compacted = number of groups written to
|
|
4142
|
+
* stage_history; deleted = number of transition rows removed.
|
|
4143
|
+
*/
|
|
4144
|
+
compact(retentionDays = this.config.retention_days) {
|
|
4145
|
+
const cutoff = new Date(Date.now() - retentionDays * MS_PER_DAY).toISOString();
|
|
4146
|
+
const oldRows = this.db.prepare(
|
|
4147
|
+
`SELECT id, work_item_id, item_type, stage, status, attempt, error_type, output, created_at
|
|
4148
|
+
FROM stage_transitions
|
|
4149
|
+
WHERE created_at < ?
|
|
4150
|
+
ORDER BY id ASC`
|
|
4151
|
+
).all(cutoff);
|
|
4152
|
+
if (oldRows.length === 0) {
|
|
4153
|
+
return { compacted: 0, deleted: 0 };
|
|
4154
|
+
}
|
|
4155
|
+
const groups = /* @__PURE__ */ new Map();
|
|
4156
|
+
for (const row of oldRows) {
|
|
4157
|
+
const key = `${row.work_item_id}\0${row.item_type}\0${row.stage}`;
|
|
4158
|
+
const existing = groups.get(key);
|
|
4159
|
+
if (existing) {
|
|
4160
|
+
existing.rows.push(row);
|
|
4161
|
+
} else {
|
|
4162
|
+
groups.set(key, { work_item_id: row.work_item_id, item_type: row.item_type, stage: row.stage, rows: [row] });
|
|
4163
|
+
}
|
|
4164
|
+
}
|
|
4165
|
+
const upsertHistory = this.db.prepare(
|
|
4166
|
+
`INSERT OR REPLACE INTO stage_history
|
|
4167
|
+
(work_item_id, item_type, stage, total_attempts, final_status, first_attempt, last_attempt, last_error, error_types, last_output)
|
|
4168
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
4169
|
+
);
|
|
4170
|
+
const deleteTransitions = this.db.prepare(
|
|
4171
|
+
"DELETE FROM stage_transitions WHERE work_item_id = ? AND item_type = ? AND stage = ? AND created_at < ?"
|
|
4172
|
+
);
|
|
4173
|
+
let compacted = 0;
|
|
4174
|
+
let deleted = 0;
|
|
4175
|
+
const doCompact = this.db.transaction(() => {
|
|
4176
|
+
for (const group of groups.values()) {
|
|
4177
|
+
const latestRow = group.rows[group.rows.length - 1];
|
|
4178
|
+
const earliestRow = group.rows[0];
|
|
4179
|
+
const errorTypeCounts = {};
|
|
4180
|
+
for (const row of group.rows) {
|
|
4181
|
+
if (row.error_type) {
|
|
4182
|
+
errorTypeCounts[row.error_type] = (errorTypeCounts[row.error_type] ?? 0) + 1;
|
|
4183
|
+
}
|
|
4184
|
+
}
|
|
4185
|
+
const lastErrorRow = this.db.prepare(
|
|
4186
|
+
`SELECT error_message FROM stage_transitions
|
|
4187
|
+
WHERE work_item_id = ? AND item_type = ? AND stage = ?
|
|
4188
|
+
ORDER BY id DESC LIMIT 1`
|
|
4189
|
+
).get(group.work_item_id, group.item_type, group.stage);
|
|
4190
|
+
const lastSucceeded = group.rows.filter((r) => r.status === "succeeded").pop();
|
|
4191
|
+
const lastOutput = lastSucceeded?.output ?? null;
|
|
4192
|
+
upsertHistory.run(
|
|
4193
|
+
group.work_item_id,
|
|
4194
|
+
group.item_type,
|
|
4195
|
+
group.stage,
|
|
4196
|
+
group.rows.length,
|
|
4197
|
+
latestRow.status,
|
|
4198
|
+
earliestRow.created_at,
|
|
4199
|
+
latestRow.created_at,
|
|
4200
|
+
lastErrorRow?.error_message ?? null,
|
|
4201
|
+
JSON.stringify(errorTypeCounts),
|
|
4202
|
+
lastOutput
|
|
4203
|
+
);
|
|
4204
|
+
const deleteResult = deleteTransitions.run(
|
|
4205
|
+
group.work_item_id,
|
|
4206
|
+
group.item_type,
|
|
4207
|
+
group.stage,
|
|
4208
|
+
cutoff
|
|
4209
|
+
);
|
|
4210
|
+
compacted++;
|
|
4211
|
+
deleted += deleteResult.changes;
|
|
4212
|
+
}
|
|
4213
|
+
});
|
|
4214
|
+
doCompact();
|
|
4215
|
+
return { compacted, deleted };
|
|
4216
|
+
}
|
|
4217
|
+
// -------------------------------------------------------------------------
|
|
4218
|
+
// Recovery
|
|
4219
|
+
// -------------------------------------------------------------------------
|
|
4220
|
+
/**
|
|
4221
|
+
* Recover stuck items on daemon startup.
|
|
4222
|
+
*
|
|
4223
|
+
* Finds all items currently in 'processing' status (via the pipeline_status view)
|
|
4224
|
+
* and inserts a new 'pending' transition for each, effectively resetting them
|
|
4225
|
+
* for reprocessing.
|
|
4226
|
+
*
|
|
4227
|
+
* Returns the count of recovered items.
|
|
4228
|
+
*/
|
|
4229
|
+
recoverStuck() {
|
|
4230
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4231
|
+
const stuckItems = this.db.prepare(
|
|
4232
|
+
`SELECT id, item_type, stage FROM pipeline_status WHERE status = 'processing'`
|
|
4233
|
+
).all();
|
|
4234
|
+
if (stuckItems.length === 0) {
|
|
4235
|
+
return 0;
|
|
4236
|
+
}
|
|
4237
|
+
const insertPending = this.db.prepare(
|
|
4238
|
+
`INSERT INTO stage_transitions
|
|
4239
|
+
(work_item_id, item_type, stage, status, attempt, error_type, error_message, started_at, completed_at, created_at)
|
|
4240
|
+
VALUES (?, ?, ?, 'pending', 1, NULL, NULL, NULL, NULL, ?)`
|
|
4241
|
+
);
|
|
4242
|
+
const doRecover = this.db.transaction(() => {
|
|
4243
|
+
for (const item of stuckItems) {
|
|
4244
|
+
insertPending.run(item.id, item.item_type, item.stage, now);
|
|
4245
|
+
}
|
|
4246
|
+
});
|
|
4247
|
+
doRecover();
|
|
4248
|
+
return stuckItems.length;
|
|
4249
|
+
}
|
|
4250
|
+
// -------------------------------------------------------------------------
|
|
4251
|
+
// API query helpers
|
|
4252
|
+
// -------------------------------------------------------------------------
|
|
4253
|
+
/**
|
|
4254
|
+
* List work items from the pipeline_status view with optional filters and pagination.
|
|
4255
|
+
*
|
|
4256
|
+
* Filters: stage, status, item_type. All optional.
|
|
4257
|
+
* Returns rows ordered by work_items.created_at DESC (newest first).
|
|
4258
|
+
*/
|
|
4259
|
+
listItems(filters) {
|
|
4260
|
+
const conditions = [];
|
|
4261
|
+
const params = [];
|
|
4262
|
+
if (filters.stage) {
|
|
4263
|
+
conditions.push("ps.stage = ?");
|
|
4264
|
+
params.push(filters.stage);
|
|
4265
|
+
}
|
|
4266
|
+
if (filters.status) {
|
|
4267
|
+
conditions.push("ps.status = ?");
|
|
4268
|
+
params.push(filters.status);
|
|
4269
|
+
}
|
|
4270
|
+
if (filters.type) {
|
|
4271
|
+
conditions.push("ps.item_type = ?");
|
|
4272
|
+
params.push(filters.type);
|
|
4273
|
+
}
|
|
4274
|
+
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
4275
|
+
const countSql = `SELECT COUNT(*) as total FROM pipeline_status ps ${where}`;
|
|
4276
|
+
const countRow = this.db.prepare(countSql).get(...params);
|
|
4277
|
+
const limit = filters.limit ?? PIPELINE_ITEMS_DEFAULT_LIMIT;
|
|
4278
|
+
const offset = filters.offset ?? 0;
|
|
4279
|
+
const querySql = `
|
|
4280
|
+
SELECT ps.id, ps.item_type, ps.source_path, ps.stage, ps.status,
|
|
4281
|
+
ps.attempt, ps.error_type, ps.error_message, ps.started_at, ps.completed_at
|
|
4282
|
+
FROM pipeline_status ps
|
|
4283
|
+
JOIN work_items wi ON wi.id = ps.id AND wi.item_type = ps.item_type
|
|
4284
|
+
${where}
|
|
4285
|
+
ORDER BY wi.created_at DESC
|
|
4286
|
+
LIMIT ? OFFSET ?
|
|
4287
|
+
`;
|
|
4288
|
+
const items = this.db.prepare(querySql).all(...params, limit, offset);
|
|
4289
|
+
return { items, total: countRow.total };
|
|
4290
|
+
}
|
|
4291
|
+
/**
|
|
4292
|
+
* Get the full transition history for a single work item.
|
|
4293
|
+
* Returns all stage_transitions rows ordered by id ASC (oldest first).
|
|
4294
|
+
*/
|
|
4295
|
+
getTransitionHistory(itemId, itemType) {
|
|
4296
|
+
return this.db.prepare(
|
|
4297
|
+
`SELECT id, work_item_id, item_type, stage, status, attempt,
|
|
4298
|
+
error_type, error_message, started_at, completed_at, output, created_at
|
|
4299
|
+
FROM stage_transitions
|
|
4300
|
+
WHERE work_item_id = ? AND item_type = ?
|
|
4301
|
+
ORDER BY id ASC`
|
|
4302
|
+
).all(itemId, itemType);
|
|
4303
|
+
}
|
|
4304
|
+
/**
|
|
4305
|
+
* Retry a single poisoned work item by inserting a new 'pending' transition
|
|
4306
|
+
* at the specified stage. Returns true if the item was poisoned and retried,
|
|
4307
|
+
* false if the item was not found or not poisoned at that stage.
|
|
4308
|
+
*/
|
|
4309
|
+
retryItem(itemId, itemType, stage) {
|
|
4310
|
+
const current = this.db.prepare(
|
|
4311
|
+
`SELECT status FROM pipeline_status
|
|
4312
|
+
WHERE id = ? AND item_type = ? AND stage = ?`
|
|
4313
|
+
).get(itemId, itemType, stage);
|
|
4314
|
+
if (!current || current.status !== "poisoned") {
|
|
4315
|
+
return false;
|
|
4316
|
+
}
|
|
4317
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4318
|
+
this.db.prepare(
|
|
4319
|
+
`INSERT INTO stage_transitions
|
|
4320
|
+
(work_item_id, item_type, stage, status, attempt, error_type, error_message, started_at, completed_at, created_at)
|
|
4321
|
+
VALUES (?, ?, ?, 'pending', 1, NULL, NULL, NULL, NULL, ?)`
|
|
4322
|
+
).run(itemId, itemType, stage, now);
|
|
4323
|
+
return true;
|
|
4324
|
+
}
|
|
4325
|
+
/**
|
|
4326
|
+
* Skip a poisoned or failed work item at the specified stage.
|
|
4327
|
+
* Inserts a 'skipped' transition so the item no longer counts as an error.
|
|
4328
|
+
* Returns true if the item was in an error state and was skipped.
|
|
4329
|
+
*/
|
|
4330
|
+
skipItem(itemId, itemType, stage) {
|
|
4331
|
+
const current = this.db.prepare(
|
|
4332
|
+
`SELECT status FROM pipeline_status
|
|
4333
|
+
WHERE id = ? AND item_type = ? AND stage = ?`
|
|
4334
|
+
).get(itemId, itemType, stage);
|
|
4335
|
+
if (!current || current.status !== "poisoned" && current.status !== "failed") {
|
|
4336
|
+
return false;
|
|
4337
|
+
}
|
|
4338
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4339
|
+
this.db.prepare(
|
|
4340
|
+
`INSERT INTO stage_transitions
|
|
4341
|
+
(work_item_id, item_type, stage, status, attempt, error_type, error_message, started_at, completed_at, created_at)
|
|
4342
|
+
VALUES (?, ?, ?, 'skipped', 1, NULL, NULL, NULL, ?, ?)`
|
|
4343
|
+
).run(itemId, itemType, stage, now, now);
|
|
4344
|
+
return true;
|
|
4345
|
+
}
|
|
4346
|
+
/**
|
|
4347
|
+
* Retry all poisoned items by inserting new 'pending' transitions.
|
|
4348
|
+
* Returns the count of items retried.
|
|
4349
|
+
*/
|
|
4350
|
+
retryAllPoisoned() {
|
|
4351
|
+
const poisonedItems = this.db.prepare(
|
|
4352
|
+
`SELECT id, item_type, stage FROM pipeline_status WHERE status = 'poisoned'`
|
|
4353
|
+
).all();
|
|
4354
|
+
if (poisonedItems.length === 0) {
|
|
4355
|
+
return 0;
|
|
4356
|
+
}
|
|
4357
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4358
|
+
const insertPending = this.db.prepare(
|
|
4359
|
+
`INSERT INTO stage_transitions
|
|
4360
|
+
(work_item_id, item_type, stage, status, attempt, error_type, error_message, started_at, completed_at, created_at)
|
|
4361
|
+
VALUES (?, ?, ?, 'pending', 1, NULL, NULL, NULL, NULL, ?)`
|
|
4362
|
+
);
|
|
4363
|
+
const doRetry = this.db.transaction(() => {
|
|
4364
|
+
for (const item of poisonedItems) {
|
|
4365
|
+
insertPending.run(item.id, item.item_type, item.stage, now);
|
|
4366
|
+
}
|
|
4367
|
+
});
|
|
4368
|
+
doRetry();
|
|
4369
|
+
return poisonedItems.length;
|
|
4370
|
+
}
|
|
4371
|
+
/**
|
|
4372
|
+
* List all circuit breaker rows from the database.
|
|
4373
|
+
*/
|
|
4374
|
+
listCircuits() {
|
|
4375
|
+
return this.db.prepare("SELECT * FROM circuit_breakers ORDER BY provider_role ASC").all();
|
|
4376
|
+
}
|
|
4377
|
+
// -------------------------------------------------------------------------
|
|
4378
|
+
// Tick processing
|
|
4379
|
+
// -------------------------------------------------------------------------
|
|
4380
|
+
handlers = null;
|
|
4381
|
+
tickInProgress = false;
|
|
4382
|
+
tickLogger = null;
|
|
4383
|
+
/** Register stage handlers called by tick(). Must be set before tick() is useful. */
|
|
4384
|
+
setHandlers(handlers) {
|
|
4385
|
+
this.handlers = handlers;
|
|
4386
|
+
}
|
|
4387
|
+
/** Set a logger for tick diagnostics. */
|
|
4388
|
+
setLogger(logger) {
|
|
4389
|
+
this.tickLogger = logger;
|
|
4390
|
+
}
|
|
4391
|
+
/**
|
|
4392
|
+
* Process one tick of the pipeline: for each tick-processable stage
|
|
4393
|
+
* (extraction, embedding, consolidation), fetch a batch of pending items
|
|
4394
|
+
* and run the corresponding handler.
|
|
4395
|
+
*
|
|
4396
|
+
* Stages are processed sequentially; items within a batch run concurrently.
|
|
4397
|
+
*
|
|
4398
|
+
* Guarded by tickInProgress — if a tick is already running, returns immediately.
|
|
4399
|
+
*/
|
|
4400
|
+
async tick(batchSize) {
|
|
4401
|
+
if (this.tickInProgress) {
|
|
4402
|
+
return;
|
|
4403
|
+
}
|
|
4404
|
+
if (!this.handlers) {
|
|
4405
|
+
return;
|
|
4406
|
+
}
|
|
4407
|
+
this.tickInProgress = true;
|
|
4408
|
+
try {
|
|
4409
|
+
for (const stage of PIPELINE_TICK_STAGES) {
|
|
4410
|
+
const providerRole = STAGE_PROVIDER_MAP[stage];
|
|
4411
|
+
if (providerRole) {
|
|
4412
|
+
const circuit = this.circuitState(providerRole);
|
|
4413
|
+
if (circuit.state === "open") {
|
|
4414
|
+
const canProbe = this.probeCircuit(providerRole);
|
|
4415
|
+
if (!canProbe) {
|
|
4416
|
+
const blocked = this.blockItemsForCircuit(providerRole);
|
|
4417
|
+
if (blocked > 0) {
|
|
4418
|
+
this.tickLogger?.("debug", "pipeline", `Circuit open for ${providerRole}, blocked ${blocked} items`, { stage, providerRole });
|
|
4419
|
+
}
|
|
4420
|
+
continue;
|
|
4421
|
+
}
|
|
4422
|
+
this.tickLogger?.("debug", "pipeline", `Circuit half-open probe for ${providerRole}`, { stage, providerRole });
|
|
4423
|
+
}
|
|
4424
|
+
}
|
|
4425
|
+
const batch = this.nextBatch(stage, batchSize);
|
|
4426
|
+
if (batch.length === 0) {
|
|
4427
|
+
continue;
|
|
4428
|
+
}
|
|
4429
|
+
const handler = this.handlers[stage];
|
|
4430
|
+
if (!handler) {
|
|
4431
|
+
continue;
|
|
4432
|
+
}
|
|
4433
|
+
await Promise.all(
|
|
4434
|
+
batch.map(async (item) => {
|
|
4435
|
+
this.advance(item.id, item.item_type, stage, "processing");
|
|
4436
|
+
try {
|
|
4437
|
+
const output = await handler(item.id, item.item_type, item.source_path);
|
|
4438
|
+
this.advance(item.id, item.item_type, stage, "succeeded", void 0, output ?? void 0);
|
|
4439
|
+
if (providerRole) {
|
|
4440
|
+
const circuitAfter = this.circuitState(providerRole);
|
|
4441
|
+
if (circuitAfter.state === "half-open") {
|
|
4442
|
+
this.resetCircuit(providerRole);
|
|
4443
|
+
const unblocked = this.unblockItemsForCircuit(providerRole);
|
|
4444
|
+
this.tickLogger?.("info", "pipeline", `Circuit closed after successful probe, unblocked ${unblocked} items`, { stage, providerRole });
|
|
4445
|
+
}
|
|
4446
|
+
}
|
|
4447
|
+
} catch (err) {
|
|
4448
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
4449
|
+
const classified = classifyError(error);
|
|
4450
|
+
this.advance(item.id, item.item_type, stage, "failed", {
|
|
4451
|
+
errorType: classified.type,
|
|
4452
|
+
errorMessage: error.message
|
|
4453
|
+
});
|
|
4454
|
+
this.tickLogger?.("warn", "pipeline", `Stage handler failed: ${stage}`, {
|
|
4455
|
+
itemId: item.id,
|
|
4456
|
+
itemType: item.item_type,
|
|
4457
|
+
errorType: classified.type,
|
|
4458
|
+
error: error.message
|
|
4459
|
+
});
|
|
4460
|
+
if (classified.type === "config" && providerRole) {
|
|
4461
|
+
const circuitBefore = this.circuitState(providerRole);
|
|
4462
|
+
if (circuitBefore.state === "half-open") {
|
|
4463
|
+
this.reopenCircuit(providerRole, error.message);
|
|
4464
|
+
const blocked = this.blockItemsForCircuit(providerRole);
|
|
4465
|
+
this.tickLogger?.("warn", "pipeline", `Half-open probe failed for ${providerRole}, re-opened with doubled cooldown, blocked ${blocked} items`, { stage, providerRole });
|
|
4466
|
+
} else {
|
|
4467
|
+
this.tripCircuit(providerRole, error.message);
|
|
4468
|
+
const circuitAfter = this.circuitState(providerRole);
|
|
4469
|
+
if (circuitAfter.state === "open") {
|
|
4470
|
+
const blocked = this.blockItemsForCircuit(providerRole);
|
|
4471
|
+
this.tickLogger?.("warn", "pipeline", `Circuit opened for ${providerRole}, blocked ${blocked} items`, { stage, providerRole });
|
|
4472
|
+
}
|
|
4473
|
+
}
|
|
4474
|
+
}
|
|
4475
|
+
}
|
|
4476
|
+
})
|
|
4477
|
+
);
|
|
4478
|
+
}
|
|
4479
|
+
} finally {
|
|
4480
|
+
this.tickInProgress = false;
|
|
4481
|
+
}
|
|
4482
|
+
}
|
|
4483
|
+
// -------------------------------------------------------------------------
|
|
4484
|
+
// Digest gating helpers
|
|
4485
|
+
// -------------------------------------------------------------------------
|
|
4486
|
+
/**
|
|
4487
|
+
* Check if any upstream stages (extraction, embedding, consolidation) have
|
|
4488
|
+
* active work items (pending, processing, or blocked — but not failed/poisoned).
|
|
4489
|
+
*
|
|
4490
|
+
* Used to gate the digest engine: digest should not run while upstream
|
|
4491
|
+
* stages still have work in flight.
|
|
4492
|
+
*/
|
|
4493
|
+
hasUpstreamWork() {
|
|
4494
|
+
const active = this.db.prepare(
|
|
4495
|
+
`SELECT COUNT(*) as cnt FROM pipeline_status
|
|
4496
|
+
WHERE stage IN ('extraction', 'embedding', 'consolidation')
|
|
4497
|
+
AND status IN ('pending', 'processing', 'blocked')`
|
|
4498
|
+
).get();
|
|
4499
|
+
if (active.cnt > 0) return true;
|
|
4500
|
+
const midPipeline = this.db.prepare(
|
|
4501
|
+
`SELECT COUNT(*) as cnt FROM pipeline_status ps1
|
|
4502
|
+
WHERE ps1.stage = 'digest' AND ps1.status = 'pending'
|
|
4503
|
+
AND EXISTS (
|
|
4504
|
+
SELECT 1 FROM pipeline_status ps2
|
|
4505
|
+
WHERE ps2.id = ps1.id AND ps2.item_type = ps1.item_type
|
|
4506
|
+
AND ps2.stage != 'digest'
|
|
4507
|
+
AND ps2.status NOT IN ('succeeded', 'skipped', 'failed', 'poisoned')
|
|
4508
|
+
)`
|
|
4509
|
+
).get();
|
|
4510
|
+
return midPipeline.cnt > 0;
|
|
4511
|
+
}
|
|
4512
|
+
/**
|
|
4513
|
+
* Count items at digest:pending where ALL other stages for that item
|
|
4514
|
+
* are in a terminal state (succeeded, skipped, failed, or poisoned).
|
|
4515
|
+
*
|
|
4516
|
+
* Used to gate digest: only run when enough processed substrate has accumulated.
|
|
4517
|
+
*/
|
|
4518
|
+
newSubstrateSinceLastDigest() {
|
|
4519
|
+
const result = this.db.prepare(
|
|
4520
|
+
`SELECT COUNT(*) as cnt FROM pipeline_status ps1
|
|
4521
|
+
WHERE ps1.stage = 'digest' AND ps1.status = 'pending'
|
|
4522
|
+
AND NOT EXISTS (
|
|
4523
|
+
SELECT 1 FROM pipeline_status ps2
|
|
4524
|
+
WHERE ps2.id = ps1.id AND ps2.item_type = ps1.item_type
|
|
4525
|
+
AND ps2.stage != 'digest'
|
|
4526
|
+
AND ps2.status NOT IN ('succeeded', 'skipped', 'failed', 'poisoned')
|
|
4527
|
+
)`
|
|
4528
|
+
).get();
|
|
4529
|
+
return result.cnt;
|
|
4530
|
+
}
|
|
4531
|
+
/**
|
|
4532
|
+
* Mark all digest:pending items as digest:succeeded after a successful cycle.
|
|
4533
|
+
* Returns the number of items advanced.
|
|
4534
|
+
*/
|
|
4535
|
+
advanceDigestItems(output) {
|
|
4536
|
+
const items = this.db.prepare(
|
|
4537
|
+
`SELECT id, item_type FROM pipeline_status
|
|
4538
|
+
WHERE stage = 'digest' AND status = 'pending'`
|
|
4539
|
+
).all();
|
|
4540
|
+
const advanceAll = this.db.transaction(() => {
|
|
4541
|
+
for (const item of items) {
|
|
4542
|
+
this.advance(item.id, item.item_type, "digest", "succeeded", void 0, output);
|
|
4543
|
+
}
|
|
4544
|
+
});
|
|
4545
|
+
advanceAll();
|
|
4546
|
+
return items.length;
|
|
4547
|
+
}
|
|
4548
|
+
// -------------------------------------------------------------------------
|
|
4549
|
+
// Rebuild from vault
|
|
4550
|
+
// -------------------------------------------------------------------------
|
|
4551
|
+
/**
|
|
4552
|
+
* Walk the vault and infer pipeline stage completion from existing data.
|
|
4553
|
+
*
|
|
4554
|
+
* Used for first-run migration: when pipeline.db is empty but the vault
|
|
4555
|
+
* already has session, spore, and artifact files from prior processing.
|
|
4556
|
+
*
|
|
4557
|
+
* Algorithm:
|
|
4558
|
+
* 1. Walk sessions/, spores/, artifacts/ directories for .md files
|
|
4559
|
+
* 2. Register each as a work item with inferred stage statuses
|
|
4560
|
+
* 3. Check vector index for embedding status
|
|
4561
|
+
* 4. Check digest trace for digest status
|
|
4562
|
+
* 5. Infer extraction status for sessions (check if spores reference them)
|
|
4563
|
+
* 6. Mark spore consolidation as pending (cannot reliably infer)
|
|
4564
|
+
*/
|
|
4565
|
+
rebuild(vaultDir, vectorIndex, digestTracePath) {
|
|
4566
|
+
const digestedIds = this.loadDigestedIds(digestTracePath);
|
|
4567
|
+
const sporeSessionIds = this.collectSporeSessionIds(vaultDir);
|
|
4568
|
+
const stages = {};
|
|
4569
|
+
let registered = 0;
|
|
4570
|
+
const bumpStage = (key) => {
|
|
4571
|
+
stages[key] = (stages[key] ?? 0) + 1;
|
|
4572
|
+
};
|
|
4573
|
+
const runRebuild2 = this.db.transaction(() => {
|
|
4574
|
+
const sessionsDir = path10.join(vaultDir, "sessions");
|
|
4575
|
+
for (const filePath of walkMarkdownFiles(sessionsDir)) {
|
|
4576
|
+
const filename = path10.basename(filePath, ".md");
|
|
4577
|
+
const itemId = filename.startsWith("session-") ? filename.slice("session-".length) : filename;
|
|
4578
|
+
const relativePath = path10.relative(vaultDir, filePath);
|
|
4579
|
+
this.register(itemId, "session", relativePath);
|
|
4580
|
+
registered++;
|
|
4581
|
+
this.advance(itemId, "session", "capture", "succeeded");
|
|
4582
|
+
bumpStage("capture:succeeded");
|
|
4583
|
+
if (sporeSessionIds.has(itemId) || sporeSessionIds.has(`session-${itemId}`)) {
|
|
4584
|
+
this.advance(itemId, "session", "extraction", "succeeded");
|
|
4585
|
+
bumpStage("extraction:succeeded");
|
|
4586
|
+
} else {
|
|
4587
|
+
bumpStage("extraction:pending");
|
|
4588
|
+
}
|
|
4589
|
+
if (vectorIndex?.has(itemId) || vectorIndex?.has(`session-${itemId}`)) {
|
|
4590
|
+
this.advance(itemId, "session", "embedding", "succeeded");
|
|
4591
|
+
bumpStage("embedding:succeeded");
|
|
4592
|
+
} else {
|
|
4593
|
+
bumpStage("embedding:pending");
|
|
4594
|
+
}
|
|
4595
|
+
if (digestedIds.has(itemId) || digestedIds.has(`session-${itemId}`)) {
|
|
4596
|
+
this.advance(itemId, "session", "digest", "succeeded");
|
|
4597
|
+
bumpStage("digest:succeeded");
|
|
4598
|
+
} else {
|
|
4599
|
+
bumpStage("digest:pending");
|
|
4600
|
+
}
|
|
4601
|
+
}
|
|
4602
|
+
const sporesDir = path10.join(vaultDir, "spores");
|
|
4603
|
+
for (const filePath of walkMarkdownFiles(sporesDir)) {
|
|
4604
|
+
const itemId = path10.basename(filePath, ".md");
|
|
4605
|
+
const relativePath = path10.relative(vaultDir, filePath);
|
|
4606
|
+
this.register(itemId, "spore", relativePath);
|
|
4607
|
+
registered++;
|
|
4608
|
+
this.advance(itemId, "spore", "capture", "succeeded");
|
|
4609
|
+
bumpStage("capture:succeeded");
|
|
4610
|
+
if (vectorIndex?.has(itemId)) {
|
|
4611
|
+
this.advance(itemId, "spore", "embedding", "succeeded");
|
|
4612
|
+
bumpStage("embedding:succeeded");
|
|
4613
|
+
} else {
|
|
4614
|
+
bumpStage("embedding:pending");
|
|
4615
|
+
}
|
|
4616
|
+
bumpStage("consolidation:pending");
|
|
4617
|
+
if (digestedIds.has(itemId)) {
|
|
4618
|
+
this.advance(itemId, "spore", "digest", "succeeded");
|
|
4619
|
+
bumpStage("digest:succeeded");
|
|
4620
|
+
} else {
|
|
4621
|
+
bumpStage("digest:pending");
|
|
4622
|
+
}
|
|
4623
|
+
}
|
|
4624
|
+
const artifactsDir = path10.join(vaultDir, "artifacts");
|
|
4625
|
+
for (const filePath of walkMarkdownFiles(artifactsDir)) {
|
|
4626
|
+
const itemId = path10.basename(filePath, ".md");
|
|
4627
|
+
const relativePath = path10.relative(vaultDir, filePath);
|
|
4628
|
+
this.register(itemId, "artifact", relativePath);
|
|
4629
|
+
registered++;
|
|
4630
|
+
this.advance(itemId, "artifact", "capture", "succeeded");
|
|
4631
|
+
bumpStage("capture:succeeded");
|
|
4632
|
+
if (vectorIndex?.has(itemId)) {
|
|
4633
|
+
this.advance(itemId, "artifact", "embedding", "succeeded");
|
|
4634
|
+
bumpStage("embedding:succeeded");
|
|
4635
|
+
} else {
|
|
4636
|
+
bumpStage("embedding:pending");
|
|
4637
|
+
}
|
|
4638
|
+
if (digestedIds.has(itemId)) {
|
|
4639
|
+
this.advance(itemId, "artifact", "digest", "succeeded");
|
|
4640
|
+
bumpStage("digest:succeeded");
|
|
4641
|
+
} else {
|
|
4642
|
+
bumpStage("digest:pending");
|
|
4643
|
+
}
|
|
4644
|
+
}
|
|
4645
|
+
});
|
|
4646
|
+
runRebuild2();
|
|
4647
|
+
return { registered, stages };
|
|
4648
|
+
}
|
|
4649
|
+
/**
|
|
4650
|
+
* Read digest trace JSONL and collect all note IDs that appear in any
|
|
4651
|
+
* substrate array. Returns an empty Set if the trace file doesn't exist.
|
|
4652
|
+
*/
|
|
4653
|
+
loadDigestedIds(tracePath) {
|
|
4654
|
+
const ids = /* @__PURE__ */ new Set();
|
|
4655
|
+
if (!tracePath) return ids;
|
|
4656
|
+
let content;
|
|
4657
|
+
try {
|
|
4658
|
+
content = fs7.readFileSync(tracePath, "utf-8").trim();
|
|
4659
|
+
} catch {
|
|
4660
|
+
return ids;
|
|
4661
|
+
}
|
|
4662
|
+
if (!content) return ids;
|
|
4663
|
+
for (const line of content.split("\n")) {
|
|
4664
|
+
try {
|
|
4665
|
+
const record = JSON.parse(line);
|
|
4666
|
+
if (record.substrate) {
|
|
4667
|
+
for (const arr of Object.values(record.substrate)) {
|
|
4668
|
+
if (Array.isArray(arr)) {
|
|
4669
|
+
for (const id of arr) {
|
|
4670
|
+
ids.add(id);
|
|
4671
|
+
}
|
|
4672
|
+
}
|
|
4673
|
+
}
|
|
4674
|
+
}
|
|
4675
|
+
} catch {
|
|
4676
|
+
}
|
|
4677
|
+
}
|
|
4678
|
+
return ids;
|
|
4679
|
+
}
|
|
4680
|
+
/**
|
|
4681
|
+
* Walk spore files and collect session IDs they reference.
|
|
4682
|
+
* Used to infer whether extraction has been completed for a session.
|
|
4683
|
+
*/
|
|
4684
|
+
collectSporeSessionIds(vaultDir) {
|
|
4685
|
+
const sessionIds = /* @__PURE__ */ new Set();
|
|
4686
|
+
const sporesDir = path10.join(vaultDir, "spores");
|
|
4687
|
+
for (const filePath of walkMarkdownFiles(sporesDir)) {
|
|
4688
|
+
try {
|
|
4689
|
+
const content = fs7.readFileSync(filePath, "utf-8");
|
|
4690
|
+
const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
|
|
4691
|
+
if (!fmMatch) continue;
|
|
4692
|
+
const sessionMatch = fmMatch[1].match(/^session:\s*(?:"\[\[)?([^"\]]+)/m);
|
|
4693
|
+
if (sessionMatch) {
|
|
4694
|
+
const rawSession = sessionMatch[1].trim();
|
|
4695
|
+
sessionIds.add(rawSession);
|
|
4696
|
+
if (rawSession.startsWith("session-")) {
|
|
4697
|
+
sessionIds.add(rawSession.slice("session-".length));
|
|
4698
|
+
}
|
|
3291
4699
|
}
|
|
4700
|
+
} catch {
|
|
3292
4701
|
}
|
|
3293
|
-
}
|
|
4702
|
+
}
|
|
4703
|
+
return sessionIds;
|
|
4704
|
+
}
|
|
4705
|
+
/** Close the database connection. */
|
|
4706
|
+
close() {
|
|
4707
|
+
this.db.close();
|
|
4708
|
+
}
|
|
4709
|
+
};
|
|
4710
|
+
|
|
4711
|
+
// src/daemon/main.ts
|
|
4712
|
+
var import_yaml2 = __toESM(require_dist(), 1);
|
|
4713
|
+
import fs8 from "fs";
|
|
4714
|
+
import path11 from "path";
|
|
4715
|
+
function indexAndEmbed(relativePath, noteId, embeddingText, metadata, deps) {
|
|
4716
|
+
indexNote(deps.index, deps.vaultDir, relativePath);
|
|
4717
|
+
if (deps.vectorIndex && embeddingText) {
|
|
4718
|
+
generateEmbedding(deps.embeddingProvider, embeddingText.slice(0, EMBEDDING_INPUT_LIMIT)).then((emb) => deps.vectorIndex.upsert(noteId, emb.embedding, metadata)).catch((err) => deps.logger.debug("embeddings", "Embedding failed", { id: noteId, error: err.message }));
|
|
3294
4719
|
}
|
|
3295
4720
|
}
|
|
3296
4721
|
async function captureArtifacts(candidates, classified, sessionId, deps, lineage) {
|
|
@@ -3325,28 +4750,28 @@ ${candidate.content}`,
|
|
|
3325
4750
|
}
|
|
3326
4751
|
}
|
|
3327
4752
|
function migrateSporeFiles(vaultDir) {
|
|
3328
|
-
const sporesDir =
|
|
3329
|
-
if (!
|
|
4753
|
+
const sporesDir = path11.join(vaultDir, "spores");
|
|
4754
|
+
if (!fs8.existsSync(sporesDir)) return 0;
|
|
3330
4755
|
let moved = 0;
|
|
3331
|
-
const entries =
|
|
4756
|
+
const entries = fs8.readdirSync(sporesDir);
|
|
3332
4757
|
for (const entry of entries) {
|
|
3333
|
-
const fullPath =
|
|
4758
|
+
const fullPath = path11.join(sporesDir, entry);
|
|
3334
4759
|
if (!entry.endsWith(".md")) continue;
|
|
3335
|
-
if (
|
|
4760
|
+
if (fs8.statSync(fullPath).isDirectory()) continue;
|
|
3336
4761
|
try {
|
|
3337
|
-
const content =
|
|
4762
|
+
const content = fs8.readFileSync(fullPath, "utf-8");
|
|
3338
4763
|
const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
|
|
3339
4764
|
if (!fmMatch) continue;
|
|
3340
|
-
const parsed =
|
|
4765
|
+
const parsed = import_yaml2.default.parse(fmMatch[1]);
|
|
3341
4766
|
const obsType = parsed.observation_type;
|
|
3342
4767
|
if (!obsType) continue;
|
|
3343
4768
|
const normalizedType = obsType.replace(/_/g, "-");
|
|
3344
|
-
const targetDir =
|
|
3345
|
-
|
|
3346
|
-
const targetPath =
|
|
3347
|
-
|
|
4769
|
+
const targetDir = path11.join(sporesDir, normalizedType);
|
|
4770
|
+
fs8.mkdirSync(targetDir, { recursive: true });
|
|
4771
|
+
const targetPath = path11.join(targetDir, entry);
|
|
4772
|
+
fs8.renameSync(fullPath, targetPath);
|
|
3348
4773
|
const now = /* @__PURE__ */ new Date();
|
|
3349
|
-
|
|
4774
|
+
fs8.utimesSync(targetPath, now, now);
|
|
3350
4775
|
moved++;
|
|
3351
4776
|
} catch {
|
|
3352
4777
|
}
|
|
@@ -3359,22 +4784,22 @@ async function main() {
|
|
|
3359
4784
|
process.stderr.write("Usage: mycod --vault <path>\n");
|
|
3360
4785
|
process.exit(1);
|
|
3361
4786
|
}
|
|
3362
|
-
const vaultDir =
|
|
4787
|
+
const vaultDir = path11.resolve(vaultArg);
|
|
3363
4788
|
const config = loadConfig(vaultDir);
|
|
3364
|
-
const logger = new DaemonLogger(
|
|
4789
|
+
const logger = new DaemonLogger(path11.join(vaultDir, "logs"), {
|
|
3365
4790
|
level: config.daemon.log_level,
|
|
3366
4791
|
maxSize: config.daemon.max_log_size
|
|
3367
4792
|
});
|
|
3368
4793
|
let uiDir = null;
|
|
3369
4794
|
{
|
|
3370
|
-
let dir =
|
|
4795
|
+
let dir = path11.dirname(new URL(import.meta.url).pathname);
|
|
3371
4796
|
for (let i = 0; i < 5; i++) {
|
|
3372
|
-
const candidate =
|
|
3373
|
-
if (
|
|
4797
|
+
const candidate = path11.join(dir, "dist", "ui");
|
|
4798
|
+
if (fs8.existsSync(path11.join(dir, "package.json")) && fs8.existsSync(candidate)) {
|
|
3374
4799
|
uiDir = candidate;
|
|
3375
4800
|
break;
|
|
3376
4801
|
}
|
|
3377
|
-
dir =
|
|
4802
|
+
dir = path11.dirname(dir);
|
|
3378
4803
|
}
|
|
3379
4804
|
}
|
|
3380
4805
|
if (uiDir) {
|
|
@@ -3382,16 +4807,8 @@ async function main() {
|
|
|
3382
4807
|
}
|
|
3383
4808
|
const server = new DaemonServer({ vaultDir, logger, uiDir: uiDir ?? void 0 });
|
|
3384
4809
|
const registry = new SessionRegistry({
|
|
3385
|
-
gracePeriod:
|
|
3386
|
-
onEmpty:
|
|
3387
|
-
logger.info("daemon", "Grace period expired, shutting down");
|
|
3388
|
-
metabolism?.stop();
|
|
3389
|
-
planWatcher.stopFileWatcher();
|
|
3390
|
-
await server.stop();
|
|
3391
|
-
vectorIndex?.close();
|
|
3392
|
-
index.close();
|
|
3393
|
-
logger.close();
|
|
3394
|
-
process.exit(0);
|
|
4810
|
+
gracePeriod: 0,
|
|
4811
|
+
onEmpty: () => {
|
|
3395
4812
|
}
|
|
3396
4813
|
});
|
|
3397
4814
|
const llmProvider = createLlmProvider(config.intelligence.llm);
|
|
@@ -3399,15 +4816,192 @@ async function main() {
|
|
|
3399
4816
|
let vectorIndex = null;
|
|
3400
4817
|
try {
|
|
3401
4818
|
const testEmbed = await embeddingProvider.embed("test");
|
|
3402
|
-
vectorIndex = new VectorIndex(
|
|
4819
|
+
vectorIndex = new VectorIndex(path11.join(vaultDir, "vectors.db"), testEmbed.dimensions);
|
|
3403
4820
|
logger.info("embeddings", "Vector index initialized", { dimensions: testEmbed.dimensions });
|
|
3404
4821
|
} catch (error) {
|
|
3405
4822
|
logger.warn("embeddings", "Vector index unavailable", { error: error.message });
|
|
3406
4823
|
}
|
|
3407
4824
|
const processor = new BufferProcessor(llmProvider, config.intelligence.llm.context_window, config.capture);
|
|
3408
4825
|
const vault = new VaultWriter(vaultDir);
|
|
3409
|
-
const index = new MycoIndex(
|
|
4826
|
+
const index = new MycoIndex(path11.join(vaultDir, "index.db"));
|
|
3410
4827
|
const lineageGraph = new LineageGraph(vaultDir);
|
|
4828
|
+
const pipeline = new PipelineManager(vaultDir, config.pipeline);
|
|
4829
|
+
const recoveredCount = pipeline.recoverStuck();
|
|
4830
|
+
if (recoveredCount > 0) {
|
|
4831
|
+
logger.info("pipeline", "Recovered stuck pipeline items", { count: recoveredCount });
|
|
4832
|
+
}
|
|
4833
|
+
if (pipeline.isEmpty()) {
|
|
4834
|
+
logger.info("pipeline", "First-run migration: rebuilding pipeline from vault");
|
|
4835
|
+
const result = pipeline.rebuild(vaultDir, vectorIndex, path11.join(vaultDir, "digest", "trace.jsonl"));
|
|
4836
|
+
logger.info("pipeline", "Pipeline rebuild complete", { registered: result.registered, stages: result.stages });
|
|
4837
|
+
}
|
|
4838
|
+
let consolidationEngine = null;
|
|
4839
|
+
let consolidationPassRanThisTick = false;
|
|
4840
|
+
pipeline.setHandlers({
|
|
4841
|
+
extraction: async (itemId, itemType, sourcePath) => {
|
|
4842
|
+
if (itemType !== "session") return;
|
|
4843
|
+
logger.info("pipeline", "Extraction started", { session_id: itemId });
|
|
4844
|
+
const fullPath = sourcePath ? path11.join(vaultDir, sourcePath) : null;
|
|
4845
|
+
if (!fullPath || !fs8.existsSync(fullPath)) {
|
|
4846
|
+
throw new Error(`Session note not found: ${sourcePath}`);
|
|
4847
|
+
}
|
|
4848
|
+
const fileContent = fs8.readFileSync(fullPath, "utf-8");
|
|
4849
|
+
const { body, frontmatter } = stripFrontmatter(fileContent);
|
|
4850
|
+
const conversationMarkdown = extractSection(body, CONVERSATION_HEADING);
|
|
4851
|
+
const user = typeof frontmatter.user === "string" && frontmatter.user ? frontmatter.user : void 0;
|
|
4852
|
+
if (!conversationMarkdown.trim()) {
|
|
4853
|
+
throw new Error(`No conversation content in session note: ${sourcePath}`);
|
|
4854
|
+
}
|
|
4855
|
+
const extractionResult = await processor.process(conversationMarkdown, itemId);
|
|
4856
|
+
if (extractionResult.degraded) {
|
|
4857
|
+
throw new Error(`Observation extraction failed for session ${itemId}`);
|
|
4858
|
+
}
|
|
4859
|
+
const { summary: narrative, title } = await processor.summarizeSession(
|
|
4860
|
+
conversationMarkdown,
|
|
4861
|
+
itemId,
|
|
4862
|
+
user
|
|
4863
|
+
);
|
|
4864
|
+
if (narrative.includes(SUMMARIZATION_FAILED_MARKER)) {
|
|
4865
|
+
throw new Error(`Summarization failed for session ${itemId}: ${narrative}`);
|
|
4866
|
+
}
|
|
4867
|
+
const written = writeObservationNotes(
|
|
4868
|
+
extractionResult.observations,
|
|
4869
|
+
itemId,
|
|
4870
|
+
vault,
|
|
4871
|
+
index,
|
|
4872
|
+
vaultDir
|
|
4873
|
+
);
|
|
4874
|
+
for (const note of written) {
|
|
4875
|
+
indexAndEmbed(
|
|
4876
|
+
note.path,
|
|
4877
|
+
note.id,
|
|
4878
|
+
`${note.observation.title}
|
|
4879
|
+
${note.observation.content}`,
|
|
4880
|
+
{ type: "spore", importance: "high", session_id: itemId },
|
|
4881
|
+
indexDeps
|
|
4882
|
+
);
|
|
4883
|
+
logger.info("pipeline", "Spore written", { type: note.observation.type, title: note.observation.title, session_id: itemId });
|
|
4884
|
+
}
|
|
4885
|
+
const fmEnd = fileContent.indexOf("---", 4);
|
|
4886
|
+
const parsedFm = import_yaml2.default.parse(fileContent.slice(4, fmEnd));
|
|
4887
|
+
parsedFm.observations_count = written.length;
|
|
4888
|
+
parsedFm.summary_tokens = estimateTokens(narrative);
|
|
4889
|
+
parsedFm.extraction_model = config.intelligence.llm.model;
|
|
4890
|
+
const fmYaml = import_yaml2.default.stringify(parsedFm, { defaultStringType: "QUOTE_DOUBLE", defaultKeyType: "PLAIN" }).trim();
|
|
4891
|
+
const updatedBody = updateTitleAndSummary(body, title, narrative);
|
|
4892
|
+
fs8.writeFileSync(fullPath, `---
|
|
4893
|
+
${fmYaml}
|
|
4894
|
+
---
|
|
4895
|
+
|
|
4896
|
+
${updatedBody}`, "utf-8");
|
|
4897
|
+
indexNote(index, vaultDir, sourcePath);
|
|
4898
|
+
for (const note of written) {
|
|
4899
|
+
pipeline.register(note.id, "spore", note.path);
|
|
4900
|
+
pipeline.advance(note.id, "spore", "capture", "succeeded");
|
|
4901
|
+
}
|
|
4902
|
+
logger.info("pipeline", "Extraction completed", {
|
|
4903
|
+
session_id: itemId,
|
|
4904
|
+
observations: written.length,
|
|
4905
|
+
title
|
|
4906
|
+
});
|
|
4907
|
+
return {
|
|
4908
|
+
observations: written.length,
|
|
4909
|
+
observation_ids: written.map((n) => n.id),
|
|
4910
|
+
summary_tokens: estimateTokens(narrative),
|
|
4911
|
+
title,
|
|
4912
|
+
model: config.intelligence.llm.model
|
|
4913
|
+
};
|
|
4914
|
+
},
|
|
4915
|
+
embedding: async (itemId, itemType, sourcePath) => {
|
|
4916
|
+
if (!vectorIndex || !embeddingProvider) {
|
|
4917
|
+
throw new Error("Embedding provider or vector index not available");
|
|
4918
|
+
}
|
|
4919
|
+
if (!sourcePath) {
|
|
4920
|
+
throw new Error(`No source path for ${itemType}/${itemId}`);
|
|
4921
|
+
}
|
|
4922
|
+
const fullPath = path11.join(vaultDir, sourcePath);
|
|
4923
|
+
if (!fs8.existsSync(fullPath)) {
|
|
4924
|
+
throw new Error(`Vault note not found: ${sourcePath}`);
|
|
4925
|
+
}
|
|
4926
|
+
const fileContent = fs8.readFileSync(fullPath, "utf-8");
|
|
4927
|
+
const { body, frontmatter } = stripFrontmatter(fileContent);
|
|
4928
|
+
let embeddableText;
|
|
4929
|
+
if (itemType === "session") {
|
|
4930
|
+
const title = typeof frontmatter.title === "string" ? frontmatter.title : "";
|
|
4931
|
+
const summary = typeof frontmatter.summary === "string" ? frontmatter.summary : "";
|
|
4932
|
+
const calloutMatch = body.match(/> \[!abstract\] Summary\n((?:> .*\n?)*)/);
|
|
4933
|
+
const narrative = calloutMatch ? calloutMatch[1].replace(/^> /gm, "").trim() : "";
|
|
4934
|
+
embeddableText = `${title}
|
|
4935
|
+
${narrative || summary}`.trim();
|
|
4936
|
+
} else {
|
|
4937
|
+
const titleMatch = body.match(/^#\s+(.+)$/m);
|
|
4938
|
+
const title = titleMatch ? titleMatch[1] : "";
|
|
4939
|
+
embeddableText = `${title}
|
|
4940
|
+
${body}`.trim();
|
|
4941
|
+
}
|
|
4942
|
+
if (!embeddableText) {
|
|
4943
|
+
logger.debug("pipeline", "No embeddable content, skipping", { id: itemId, type: itemType });
|
|
4944
|
+
return;
|
|
4945
|
+
}
|
|
4946
|
+
const result = await generateEmbedding(
|
|
4947
|
+
embeddingProvider,
|
|
4948
|
+
embeddableText.slice(0, EMBEDDING_INPUT_LIMIT)
|
|
4949
|
+
);
|
|
4950
|
+
const metadata = {
|
|
4951
|
+
type: itemType,
|
|
4952
|
+
file_path: sourcePath,
|
|
4953
|
+
session_id: typeof frontmatter.session === "string" ? frontmatter.session : typeof frontmatter.id === "string" && itemType === "session" ? frontmatter.id : "",
|
|
4954
|
+
branch: typeof frontmatter.branch === "string" ? frontmatter.branch : ""
|
|
4955
|
+
};
|
|
4956
|
+
vectorIndex.upsert(itemId, result.embedding, metadata);
|
|
4957
|
+
if (itemType === "session" && sourcePath) {
|
|
4958
|
+
vault.updateNoteFrontmatter(sourcePath, {
|
|
4959
|
+
embedding_model: config.intelligence.embedding.model
|
|
4960
|
+
}, true);
|
|
4961
|
+
}
|
|
4962
|
+
logger.info("pipeline", "Embedding stored", {
|
|
4963
|
+
id: itemId,
|
|
4964
|
+
type: itemType,
|
|
4965
|
+
dimensions: result.dimensions
|
|
4966
|
+
});
|
|
4967
|
+
return {
|
|
4968
|
+
model: result.model,
|
|
4969
|
+
dimensions: result.dimensions
|
|
4970
|
+
};
|
|
4971
|
+
},
|
|
4972
|
+
consolidation: async (itemId, itemType) => {
|
|
4973
|
+
if (itemType !== "spore") return;
|
|
4974
|
+
const supersededIds = await checkSupersession(itemId, {
|
|
4975
|
+
index,
|
|
4976
|
+
vectorIndex,
|
|
4977
|
+
embeddingProvider,
|
|
4978
|
+
llmProvider,
|
|
4979
|
+
vaultDir,
|
|
4980
|
+
log: ((level, msg, data) => logger[level]("curation", msg, data))
|
|
4981
|
+
});
|
|
4982
|
+
if (consolidationEngine && !consolidationPassRanThisTick) {
|
|
4983
|
+
consolidationPassRanThisTick = true;
|
|
4984
|
+
await consolidationEngine.runPass();
|
|
4985
|
+
}
|
|
4986
|
+
return {
|
|
4987
|
+
superseded: supersededIds ?? [],
|
|
4988
|
+
wisdom_created: null,
|
|
4989
|
+
candidates_evaluated: 0
|
|
4990
|
+
};
|
|
4991
|
+
}
|
|
4992
|
+
});
|
|
4993
|
+
pipeline.setLogger((level, domain, message, data) => {
|
|
4994
|
+
const fn = logger[level];
|
|
4995
|
+
if (typeof fn === "function") fn.call(logger, domain, message, data);
|
|
4996
|
+
});
|
|
4997
|
+
const pipelineTickTimer = setInterval(async () => {
|
|
4998
|
+
try {
|
|
4999
|
+
consolidationPassRanThisTick = false;
|
|
5000
|
+
await pipeline.tick(config.pipeline.batch_size);
|
|
5001
|
+
} catch (err) {
|
|
5002
|
+
logger.error("pipeline", "Pipeline tick failed", { error: err.message });
|
|
5003
|
+
}
|
|
5004
|
+
}, config.pipeline.tick_interval_seconds * 1e3);
|
|
3411
5005
|
const transcriptMiner = new TranscriptMiner({
|
|
3412
5006
|
additionalAdapters: config.capture.transcript_paths.map(
|
|
3413
5007
|
(p) => createPerProjectAdapter(p, claudeCodeAdapter.parseTurns)
|
|
@@ -3415,17 +5009,17 @@ async function main() {
|
|
|
3415
5009
|
});
|
|
3416
5010
|
let activeStopProcessing = null;
|
|
3417
5011
|
const indexDeps = { index, vaultDir, vectorIndex, embeddingProvider, llmProvider, logger };
|
|
3418
|
-
const bufferDir =
|
|
5012
|
+
const bufferDir = path11.join(vaultDir, "buffer");
|
|
3419
5013
|
const sessionBuffers = /* @__PURE__ */ new Map();
|
|
3420
5014
|
const sessionFilePaths = /* @__PURE__ */ new Map();
|
|
3421
5015
|
const capturedArtifactPaths = /* @__PURE__ */ new Map();
|
|
3422
|
-
if (
|
|
5016
|
+
if (fs8.existsSync(bufferDir)) {
|
|
3423
5017
|
const cutoff = Date.now() - STALE_BUFFER_MAX_AGE_MS;
|
|
3424
|
-
for (const file of
|
|
3425
|
-
const filePath =
|
|
3426
|
-
const stat4 =
|
|
5018
|
+
for (const file of fs8.readdirSync(bufferDir)) {
|
|
5019
|
+
const filePath = path11.join(bufferDir, file);
|
|
5020
|
+
const stat4 = fs8.statSync(filePath);
|
|
3427
5021
|
if (stat4.mtimeMs < cutoff) {
|
|
3428
|
-
|
|
5022
|
+
fs8.unlinkSync(filePath);
|
|
3429
5023
|
logger.debug("daemon", "Cleaned stale buffer", { file });
|
|
3430
5024
|
}
|
|
3431
5025
|
}
|
|
@@ -3462,10 +5056,10 @@ async function main() {
|
|
|
3462
5056
|
logger.info("watcher", "Plan detected", { source: event.source, file: event.filePath });
|
|
3463
5057
|
if (event.filePath) {
|
|
3464
5058
|
try {
|
|
3465
|
-
const content =
|
|
3466
|
-
const relativePath =
|
|
3467
|
-
const title = content.match(/^#\s+(.+)$/m)?.[1] ??
|
|
3468
|
-
const planId = `plan-${
|
|
5059
|
+
const content = fs8.readFileSync(event.filePath, "utf-8");
|
|
5060
|
+
const relativePath = path11.relative(vaultDir, event.filePath);
|
|
5061
|
+
const title = content.match(/^#\s+(.+)$/m)?.[1] ?? path11.basename(event.filePath);
|
|
5062
|
+
const planId = `plan-${path11.basename(event.filePath, ".md")}`;
|
|
3469
5063
|
indexAndEmbed(
|
|
3470
5064
|
relativePath,
|
|
3471
5065
|
planId,
|
|
@@ -3486,16 +5080,16 @@ ${content}`,
|
|
|
3486
5080
|
}
|
|
3487
5081
|
});
|
|
3488
5082
|
planWatcher.startFileWatcher();
|
|
5083
|
+
const digestLlmConfig = {
|
|
5084
|
+
provider: config.digest.intelligence.provider ?? config.intelligence.llm.provider,
|
|
5085
|
+
model: config.digest.intelligence.model ?? config.intelligence.llm.model,
|
|
5086
|
+
base_url: config.digest.intelligence.base_url ?? config.intelligence.llm.base_url,
|
|
5087
|
+
context_window: config.digest.intelligence.context_window
|
|
5088
|
+
};
|
|
5089
|
+
const digestLlm = config.digest.intelligence.model || config.digest.intelligence.provider ? createLlmProvider(digestLlmConfig) : llmProvider;
|
|
3489
5090
|
let metabolism = null;
|
|
3490
5091
|
if (config.digest.enabled) {
|
|
3491
|
-
const digestLlmConfig = {
|
|
3492
|
-
provider: config.digest.intelligence.provider ?? config.intelligence.llm.provider,
|
|
3493
|
-
model: config.digest.intelligence.model ?? config.intelligence.llm.model,
|
|
3494
|
-
base_url: config.digest.intelligence.base_url ?? config.intelligence.llm.base_url,
|
|
3495
|
-
context_window: config.digest.intelligence.context_window
|
|
3496
|
-
};
|
|
3497
5092
|
logger.debug("digest", "Digest LLM config", digestLlmConfig);
|
|
3498
|
-
const digestLlm = config.digest.intelligence.model || config.digest.intelligence.provider ? createLlmProvider(digestLlmConfig) : llmProvider;
|
|
3499
5093
|
logger.debug("digest", `Using ${digestLlm.name} provider for digest`);
|
|
3500
5094
|
const digestEngine = new DigestEngine({
|
|
3501
5095
|
vaultDir,
|
|
@@ -3505,7 +5099,7 @@ ${content}`,
|
|
|
3505
5099
|
log: (level, message, data) => logger[level]("digest", message, data)
|
|
3506
5100
|
});
|
|
3507
5101
|
if (config.digest.consolidation.enabled) {
|
|
3508
|
-
|
|
5102
|
+
consolidationEngine = new ConsolidationEngine({
|
|
3509
5103
|
vaultDir,
|
|
3510
5104
|
index,
|
|
3511
5105
|
vectorIndex,
|
|
@@ -3514,55 +5108,74 @@ ${content}`,
|
|
|
3514
5108
|
maxTokens: config.digest.consolidation.max_tokens,
|
|
3515
5109
|
log: (level, message, data) => logger[level]("consolidation", message, data)
|
|
3516
5110
|
});
|
|
3517
|
-
|
|
3518
|
-
const result = await consolidationEngine.runPass();
|
|
3519
|
-
if (result && result.consolidated > 0) {
|
|
3520
|
-
logger.info("consolidation", `Consolidation pass: ${result.consolidated} wisdom notes, ${result.sporesSuperseded} spores superseded`);
|
|
3521
|
-
}
|
|
3522
|
-
});
|
|
3523
|
-
logger.info("consolidation", "Auto-consolidation enabled as digest pre-pass");
|
|
5111
|
+
logger.info("consolidation", "Auto-consolidation enabled as pipeline stage");
|
|
3524
5112
|
}
|
|
3525
5113
|
metabolism = new Metabolism(config.digest.metabolism);
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
if (result) {
|
|
3529
|
-
metabolism.onSubstrateFound();
|
|
3530
|
-
logger.info("digest", `Initial digest cycle: ${result.tiersGenerated.length} tiers, ${result.durationMs}ms`);
|
|
3531
|
-
}
|
|
3532
|
-
}).catch((err) => {
|
|
3533
|
-
logger.warn("digest", "Initial digest cycle failed", { error: err.message });
|
|
3534
|
-
metabolism.onEmptyCycle();
|
|
3535
|
-
});
|
|
5114
|
+
let digestReady = true;
|
|
5115
|
+
let forceDigest = false;
|
|
3536
5116
|
metabolism.start(async () => {
|
|
5117
|
+
digestReady = true;
|
|
5118
|
+
});
|
|
5119
|
+
const originalTick = pipeline.tick.bind(pipeline);
|
|
5120
|
+
pipeline.tick = async (batchSize) => {
|
|
5121
|
+
await originalTick(batchSize);
|
|
5122
|
+
if (!digestReady) return;
|
|
5123
|
+
if (pipeline.hasUpstreamWork()) {
|
|
5124
|
+
logger.debug("digest", "Digest deferred \u2014 upstream stages pending");
|
|
5125
|
+
return;
|
|
5126
|
+
}
|
|
5127
|
+
if (!forceDigest) {
|
|
5128
|
+
const ready = pipeline.newSubstrateSinceLastDigest();
|
|
5129
|
+
if (ready < config.digest.substrate.min_notes_for_cycle) {
|
|
5130
|
+
logger.debug("digest", "Digest deferred \u2014 insufficient substrate", {
|
|
5131
|
+
ready,
|
|
5132
|
+
threshold: config.digest.substrate.min_notes_for_cycle
|
|
5133
|
+
});
|
|
5134
|
+
return;
|
|
5135
|
+
}
|
|
5136
|
+
}
|
|
5137
|
+
digestReady = false;
|
|
5138
|
+
forceDigest = false;
|
|
3537
5139
|
try {
|
|
3538
5140
|
const cycleResult = await digestEngine.runCycle();
|
|
3539
5141
|
if (cycleResult) {
|
|
3540
5142
|
metabolism.onSubstrateFound();
|
|
5143
|
+
const digestOutput = {
|
|
5144
|
+
included_in_cycle: cycleResult.cycleId,
|
|
5145
|
+
tiers_generated: cycleResult.tiersGenerated
|
|
5146
|
+
};
|
|
5147
|
+
const advanced = pipeline.advanceDigestItems(digestOutput);
|
|
5148
|
+
if (advanced > 0) {
|
|
5149
|
+
logger.debug("digest", `Advanced ${advanced} pipeline items to digest:succeeded`);
|
|
5150
|
+
}
|
|
3541
5151
|
logger.info("digest", `Digest cycle ${cycleResult.cycleId}: ${cycleResult.tiersGenerated.length} tiers`);
|
|
3542
5152
|
} else {
|
|
3543
5153
|
metabolism.onEmptyCycle();
|
|
3544
|
-
logger.debug("digest", "No substrate, backing off");
|
|
3545
5154
|
}
|
|
3546
5155
|
} catch (err) {
|
|
3547
5156
|
logger.warn("digest", "Digest cycle failed", { error: err.message });
|
|
3548
5157
|
metabolism.onEmptyCycle();
|
|
3549
5158
|
}
|
|
3550
|
-
}
|
|
3551
|
-
|
|
5159
|
+
};
|
|
5160
|
+
server.registerRoute("POST", "/api/pipeline/digest/force", handleForceDigest(() => {
|
|
5161
|
+
forceDigest = true;
|
|
5162
|
+
}));
|
|
5163
|
+
server.registerRoute("GET", "/api/pipeline/digest-health", handleDigestHealth({
|
|
5164
|
+
vaultDir,
|
|
5165
|
+
pipeline,
|
|
5166
|
+
minNotesForCycle: config.digest.substrate.min_notes_for_cycle,
|
|
5167
|
+
metabolismState: () => metabolism?.state ?? "disabled",
|
|
5168
|
+
digestReady: () => digestReady,
|
|
5169
|
+
cycleInProgress: () => digestEngine.isCycleInProgress
|
|
5170
|
+
}));
|
|
5171
|
+
logger.info("digest", "Digest enabled \u2014 controlled by pipeline tick");
|
|
3552
5172
|
}
|
|
3553
5173
|
const batchManager = new BatchManager(async (closedBatch) => {
|
|
3554
5174
|
if (closedBatch.length === 0) return;
|
|
3555
5175
|
const sessionId = closedBatch[0].session_id;
|
|
3556
|
-
|
|
3557
|
-
const result = await processor.process(asRecords, sessionId);
|
|
3558
|
-
if (!result.degraded) {
|
|
3559
|
-
writeObservations(result.observations, sessionId, { vault, ...indexDeps });
|
|
3560
|
-
}
|
|
3561
|
-
logger.debug("processor", "Batch processed", {
|
|
5176
|
+
logger.debug("processor", "Batch closed (extraction deferred to pipeline)", {
|
|
3562
5177
|
session_id: sessionId,
|
|
3563
|
-
events: closedBatch.length
|
|
3564
|
-
observations: result.observations.length,
|
|
3565
|
-
degraded: result.degraded
|
|
5178
|
+
events: closedBatch.length
|
|
3566
5179
|
});
|
|
3567
5180
|
const allPaths = sessionFilePaths.get(sessionId);
|
|
3568
5181
|
const alreadyCaptured = capturedArtifactPaths.get(sessionId) ?? /* @__PURE__ */ new Set();
|
|
@@ -3580,7 +5193,7 @@ ${content}`,
|
|
|
3580
5193
|
}
|
|
3581
5194
|
const captured = capturedArtifactPaths.get(sessionId);
|
|
3582
5195
|
for (const c of candidates) {
|
|
3583
|
-
const absPath =
|
|
5196
|
+
const absPath = path11.resolve(process.cwd(), c.path);
|
|
3584
5197
|
captured.add(absPath);
|
|
3585
5198
|
}
|
|
3586
5199
|
}).catch((err) => logger.warn("processor", "Incremental artifact capture failed", {
|
|
@@ -3620,14 +5233,14 @@ ${content}`,
|
|
|
3620
5233
|
registry.unregister(session_id);
|
|
3621
5234
|
try {
|
|
3622
5235
|
const cutoff = Date.now() - STALE_BUFFER_MAX_AGE_MS;
|
|
3623
|
-
for (const file of
|
|
5236
|
+
for (const file of fs8.readdirSync(bufferDir)) {
|
|
3624
5237
|
if (!file.endsWith(".jsonl")) continue;
|
|
3625
5238
|
const bufferSessionId = file.replace(".jsonl", "");
|
|
3626
5239
|
if (bufferSessionId === session_id) continue;
|
|
3627
|
-
const filePath =
|
|
3628
|
-
const stat4 =
|
|
5240
|
+
const filePath = path11.join(bufferDir, file);
|
|
5241
|
+
const stat4 = fs8.statSync(filePath);
|
|
3629
5242
|
if (stat4.mtimeMs < cutoff) {
|
|
3630
|
-
|
|
5243
|
+
fs8.unlinkSync(filePath);
|
|
3631
5244
|
logger.debug("daemon", "Cleaned stale buffer", { file });
|
|
3632
5245
|
}
|
|
3633
5246
|
}
|
|
@@ -3687,6 +5300,32 @@ ${content}`,
|
|
|
3687
5300
|
});
|
|
3688
5301
|
return { body: { ok: true } };
|
|
3689
5302
|
});
|
|
5303
|
+
function enrichTurnsWithToolMetadata(turns, events) {
|
|
5304
|
+
if (events.length === 0 || turns.length === 0) return;
|
|
5305
|
+
const toolEvents = events.filter((e) => e.type === "tool_use");
|
|
5306
|
+
if (toolEvents.length === 0) return;
|
|
5307
|
+
let cursor = 0;
|
|
5308
|
+
for (let i = 0; i < turns.length; i++) {
|
|
5309
|
+
const turnEnd = i + 1 < turns.length ? turns[i + 1].timestamp : null;
|
|
5310
|
+
const breakdown = {};
|
|
5311
|
+
const files = /* @__PURE__ */ new Set();
|
|
5312
|
+
while (cursor < toolEvents.length) {
|
|
5313
|
+
const ts = String(toolEvents[cursor].timestamp ?? "");
|
|
5314
|
+
if (turnEnd !== null && ts >= turnEnd) break;
|
|
5315
|
+
const evt = toolEvents[cursor];
|
|
5316
|
+
const toolName = String(evt.tool_name ?? evt.tool ?? "unknown");
|
|
5317
|
+
breakdown[toolName] = (breakdown[toolName] ?? 0) + 1;
|
|
5318
|
+
const input = evt.tool_input;
|
|
5319
|
+
const filePath = input?.file_path ?? input?.path;
|
|
5320
|
+
if (typeof filePath === "string") files.add(filePath);
|
|
5321
|
+
cursor++;
|
|
5322
|
+
}
|
|
5323
|
+
if (Object.keys(breakdown).length > 0) {
|
|
5324
|
+
turns[i].toolBreakdown = breakdown;
|
|
5325
|
+
if (files.size > 0) turns[i].files = [...files];
|
|
5326
|
+
}
|
|
5327
|
+
}
|
|
5328
|
+
}
|
|
3690
5329
|
async function processStopEvent(sessionId, user, sessionMeta, hookTranscriptPath, lastAssistantMessage) {
|
|
3691
5330
|
const lastBatch = batchManager.finalize(sessionId);
|
|
3692
5331
|
const transcriptResult = transcriptMiner.getAllTurnsWithSource(sessionId, hookTranscriptPath);
|
|
@@ -3720,17 +5359,18 @@ ${content}`,
|
|
|
3720
5359
|
lastTurn.aiResponse = lastAssistantMessage;
|
|
3721
5360
|
}
|
|
3722
5361
|
}
|
|
5362
|
+
enrichTurnsWithToolMetadata(allTurns, bufferEvents);
|
|
3723
5363
|
const ended = (/* @__PURE__ */ new Date()).toISOString();
|
|
3724
5364
|
let started = allTurns.length > 0 && allTurns[0].timestamp ? allTurns[0].timestamp : ended;
|
|
3725
|
-
const sessionsDir =
|
|
5365
|
+
const sessionsDir = path11.join(vaultDir, "sessions");
|
|
3726
5366
|
const sessionFileName = `${sessionNoteId(sessionId)}.md`;
|
|
3727
5367
|
let existingContent;
|
|
3728
5368
|
const duplicatePaths = [];
|
|
3729
5369
|
try {
|
|
3730
|
-
for (const dateDir of
|
|
3731
|
-
const candidate =
|
|
5370
|
+
for (const dateDir of fs8.readdirSync(sessionsDir)) {
|
|
5371
|
+
const candidate = path11.join(sessionsDir, dateDir, sessionFileName);
|
|
3732
5372
|
try {
|
|
3733
|
-
const content =
|
|
5373
|
+
const content = fs8.readFileSync(candidate, "utf-8");
|
|
3734
5374
|
if (!existingContent || content.length > existingContent.length) {
|
|
3735
5375
|
existingContent = content;
|
|
3736
5376
|
}
|
|
@@ -3741,11 +5381,18 @@ ${content}`,
|
|
|
3741
5381
|
} catch {
|
|
3742
5382
|
}
|
|
3743
5383
|
let existingTurnCount = 0;
|
|
5384
|
+
let existingExtractionFields = null;
|
|
3744
5385
|
if (existingContent) {
|
|
3745
5386
|
const fmMatch = existingContent.match(/^---\n([\s\S]*?)\n---/);
|
|
3746
5387
|
if (fmMatch) {
|
|
3747
|
-
const parsed =
|
|
5388
|
+
const parsed = import_yaml2.default.parse(fmMatch[1]);
|
|
3748
5389
|
if (typeof parsed.started === "string") started = parsed.started;
|
|
5390
|
+
const extractionKeys = ["observations_count", "summary_tokens", "extraction_model", "embedding_model"];
|
|
5391
|
+
const preserved = {};
|
|
5392
|
+
for (const key of extractionKeys) {
|
|
5393
|
+
if (parsed[key] !== void 0) preserved[key] = parsed[key];
|
|
5394
|
+
}
|
|
5395
|
+
if (Object.keys(preserved).length > 0) existingExtractionFields = preserved;
|
|
3749
5396
|
}
|
|
3750
5397
|
const turnMatches = existingContent.match(/^### Turn \d+/gm);
|
|
3751
5398
|
existingTurnCount = turnMatches?.length ?? 0;
|
|
@@ -3766,47 +5413,22 @@ ${content}`,
|
|
|
3766
5413
|
logger.debug("processor", "No new turns, skipping session rewrite", { session_id: sessionId, turns: allTurns.length });
|
|
3767
5414
|
return;
|
|
3768
5415
|
}
|
|
3769
|
-
const conversationText = allTurns.map((t, i) => {
|
|
3770
|
-
const parts = [`### Turn ${i + 1}`];
|
|
3771
|
-
if (t.prompt) parts.push(`Prompt: ${t.prompt}`);
|
|
3772
|
-
if (t.toolCount > 0) parts.push(`Tools: ${t.toolCount} calls`);
|
|
3773
|
-
if (t.aiResponse) parts.push(`Response: ${t.aiResponse}`);
|
|
3774
|
-
return parts.join("\n");
|
|
3775
|
-
}).join("\n\n");
|
|
3776
|
-
const conversationSection = `${CONVERSATION_HEADING}
|
|
3777
|
-
|
|
3778
|
-
${conversationText}`;
|
|
3779
|
-
const observationPromise = lastBatch.length > 0 ? processor.process(lastBatch, sessionId).catch((err) => {
|
|
3780
|
-
logger.warn("processor", "Observation extraction failed", { session_id: sessionId, error: err.message });
|
|
3781
|
-
return null;
|
|
3782
|
-
}) : Promise.resolve(null);
|
|
3783
|
-
const artifactPromise = artifactCandidates.length > 0 ? processor.classifyArtifacts(artifactCandidates, sessionId).then((classified) => captureArtifacts(artifactCandidates, classified, sessionId, { vault, ...indexDeps }, lineageGraph)).catch((err) => {
|
|
3784
|
-
logger.warn("processor", "Artifact capture failed", { session_id: sessionId, error: err.message });
|
|
3785
|
-
}) : Promise.resolve();
|
|
3786
|
-
const summaryPromise = processor.summarizeSession(conversationSection, sessionId, user).catch((err) => {
|
|
3787
|
-
logger.warn("processor", "Session summarization failed", { session_id: sessionId, error: err.message });
|
|
3788
|
-
return null;
|
|
3789
|
-
});
|
|
3790
|
-
const [observationResult, , summaryResult] = await Promise.all([observationPromise, artifactPromise, summaryPromise]);
|
|
3791
|
-
if (observationResult && !observationResult.degraded) {
|
|
3792
|
-
writeObservations(observationResult.observations, sessionId, { vault, ...indexDeps });
|
|
3793
|
-
}
|
|
3794
5416
|
const date = started.slice(0, 10);
|
|
3795
5417
|
const relativePath = sessionRelativePath(sessionId, date);
|
|
3796
|
-
const targetFullPath =
|
|
5418
|
+
const targetFullPath = path11.join(vaultDir, relativePath);
|
|
3797
5419
|
for (const dup of duplicatePaths) {
|
|
3798
5420
|
if (dup !== targetFullPath) {
|
|
3799
5421
|
try {
|
|
3800
|
-
|
|
5422
|
+
fs8.unlinkSync(dup);
|
|
3801
5423
|
logger.debug("lifecycle", "Removed duplicate session file", { path: dup });
|
|
3802
5424
|
} catch {
|
|
3803
5425
|
}
|
|
3804
5426
|
}
|
|
3805
5427
|
}
|
|
3806
|
-
const attachmentsDir =
|
|
5428
|
+
const attachmentsDir = path11.join(vaultDir, "attachments");
|
|
3807
5429
|
const hasImages = allTurns.some((t) => t.images?.length);
|
|
3808
5430
|
if (hasImages) {
|
|
3809
|
-
|
|
5431
|
+
fs8.mkdirSync(attachmentsDir, { recursive: true });
|
|
3810
5432
|
}
|
|
3811
5433
|
const turnImageNames = /* @__PURE__ */ new Map();
|
|
3812
5434
|
for (let i = 0; i < allTurns.length; i++) {
|
|
@@ -3817,9 +5439,9 @@ ${conversationText}`;
|
|
|
3817
5439
|
const img = turn.images[j];
|
|
3818
5440
|
const ext = extensionForMimeType(img.mediaType);
|
|
3819
5441
|
const filename = `${bareSessionId(sessionId)}-t${i + 1}-${j + 1}.${ext}`;
|
|
3820
|
-
const filePath =
|
|
3821
|
-
if (!
|
|
3822
|
-
|
|
5442
|
+
const filePath = path11.join(attachmentsDir, filename);
|
|
5443
|
+
if (!fs8.existsSync(filePath)) {
|
|
5444
|
+
fs8.writeFileSync(filePath, Buffer.from(img.data, "base64"));
|
|
3823
5445
|
logger.debug("processor", "Image saved", { filename, turn: i + 1 });
|
|
3824
5446
|
}
|
|
3825
5447
|
names.push(filename);
|
|
@@ -3828,9 +5450,15 @@ ${conversationText}`;
|
|
|
3828
5450
|
}
|
|
3829
5451
|
let title = `Session ${sessionId}`;
|
|
3830
5452
|
let narrative = "";
|
|
3831
|
-
if (
|
|
3832
|
-
|
|
3833
|
-
|
|
5453
|
+
if (existingContent) {
|
|
5454
|
+
const existingTitle = existingContent.match(/^# (.+)$/m)?.[1];
|
|
5455
|
+
if (existingTitle && existingTitle !== `Session ${sessionId}`) {
|
|
5456
|
+
title = existingTitle;
|
|
5457
|
+
}
|
|
5458
|
+
const calloutMatch = existingContent.match(/> \[!abstract\] Summary\n((?:> .*\n?)*)/);
|
|
5459
|
+
if (calloutMatch) {
|
|
5460
|
+
narrative = calloutMatch[1].replace(/^> /gm, "").trim();
|
|
5461
|
+
}
|
|
3834
5462
|
}
|
|
3835
5463
|
const relatedMemories = index.query({ type: "spore", limit: RELATED_SPORES_LIMIT }).filter((n) => {
|
|
3836
5464
|
const fm = n.frontmatter;
|
|
@@ -3848,6 +5476,8 @@ ${conversationText}`;
|
|
|
3848
5476
|
turns: allTurns.map((t, i) => ({
|
|
3849
5477
|
prompt: t.prompt,
|
|
3850
5478
|
toolCount: t.toolCount,
|
|
5479
|
+
toolBreakdown: t.toolBreakdown,
|
|
5480
|
+
files: t.files,
|
|
3851
5481
|
aiResponse: t.aiResponse,
|
|
3852
5482
|
images: turnImageNames.get(i)
|
|
3853
5483
|
}))
|
|
@@ -3863,62 +5493,18 @@ ${conversationText}`;
|
|
|
3863
5493
|
parent: parentId ? sessionWikilink(parentId) : void 0,
|
|
3864
5494
|
parent_reason: parentLink?.signal,
|
|
3865
5495
|
tools_used: allTurns.reduce((sum, t) => sum + t.toolCount, 0),
|
|
5496
|
+
transcript_source: turnSource,
|
|
5497
|
+
transcript_path: hookTranscriptPath,
|
|
3866
5498
|
summary
|
|
3867
5499
|
});
|
|
3868
|
-
|
|
3869
|
-
relativePath,
|
|
3870
|
-
sessionNoteId(sessionId),
|
|
3871
|
-
narrative,
|
|
3872
|
-
{ type: "session", session_id: sessionId },
|
|
3873
|
-
indexDeps
|
|
3874
|
-
);
|
|
3875
|
-
logger.debug("processor", "Session turns", { source: turnSource, total: allTurns.length });
|
|
3876
|
-
await artifactPromise;
|
|
3877
|
-
if (!parentId && vectorIndex && narrative) {
|
|
3878
|
-
generateEmbedding(embeddingProvider, narrative).then(async (emb) => {
|
|
3879
|
-
const candidates = vectorIndex.search(emb.embedding, { limit: LINEAGE_SIMILARITY_CANDIDATES }).filter((r) => r.metadata.type === "session" && r.id !== sessionNoteId(sessionId));
|
|
3880
|
-
if (candidates.length === 0) return;
|
|
3881
|
-
const candidateNotes = index.queryByIds(candidates.map((c) => c.id));
|
|
3882
|
-
const noteMap = new Map(candidateNotes.map((n) => [n.id, n]));
|
|
3883
|
-
const scores = await Promise.all(candidates.map(async (candidate) => {
|
|
3884
|
-
const note = noteMap.get(candidate.id);
|
|
3885
|
-
if (!note) return { id: candidate.id, score: 0 };
|
|
3886
|
-
try {
|
|
3887
|
-
const prompt = buildSimilarityPrompt(narrative, note.content.slice(0, CANDIDATE_CONTENT_PREVIEW));
|
|
3888
|
-
const response = await llmProvider.summarize(prompt, { maxTokens: LINEAGE_SIMILARITY_MAX_TOKENS, reasoning: LLM_REASONING_MODE });
|
|
3889
|
-
const score = extractNumber(response.text);
|
|
3890
|
-
return { id: candidate.id, score: isNaN(score) ? 0 : score };
|
|
3891
|
-
} catch {
|
|
3892
|
-
return { id: candidate.id, score: 0 };
|
|
3893
|
-
}
|
|
3894
|
-
}));
|
|
3895
|
-
const best = scores.reduce((a, b) => b.score > a.score ? b : a);
|
|
3896
|
-
if (best.score >= LINEAGE_SIMILARITY_THRESHOLD) {
|
|
3897
|
-
const bestParentId = bareSessionId(best.id);
|
|
3898
|
-
const confidence = best.score >= LINEAGE_SIMILARITY_HIGH_CONFIDENCE ? "high" : "medium";
|
|
3899
|
-
lineageGraph.addLink({
|
|
3900
|
-
parent: bestParentId,
|
|
3901
|
-
child: sessionId,
|
|
3902
|
-
signal: "semantic_similarity",
|
|
3903
|
-
confidence
|
|
3904
|
-
});
|
|
3905
|
-
try {
|
|
3906
|
-
vault.updateNoteFrontmatter(relativePath, {
|
|
3907
|
-
parent: sessionWikilink(bestParentId),
|
|
3908
|
-
parent_reason: "semantic_similarity"
|
|
3909
|
-
});
|
|
3910
|
-
indexNote(index, vaultDir, relativePath);
|
|
3911
|
-
} catch {
|
|
3912
|
-
}
|
|
3913
|
-
logger.info("lineage", "LLM similarity parent detected", {
|
|
3914
|
-
child: sessionId,
|
|
3915
|
-
parent: bestParentId,
|
|
3916
|
-
score: best.score
|
|
3917
|
-
});
|
|
3918
|
-
}
|
|
3919
|
-
}).catch((err) => logger.debug("lineage", "Similarity detection failed", { error: err.message }));
|
|
5500
|
+
if (existingExtractionFields) {
|
|
5501
|
+
vault.updateNoteFrontmatter(relativePath, existingExtractionFields);
|
|
3920
5502
|
}
|
|
3921
|
-
|
|
5503
|
+
indexNote(index, vaultDir, relativePath);
|
|
5504
|
+
logger.debug("processor", "Session turns", { source: turnSource, total: allTurns.length });
|
|
5505
|
+
pipeline.register(sessionId, "session", relativePath);
|
|
5506
|
+
pipeline.advance(sessionId, "session", "capture", "succeeded");
|
|
5507
|
+
logger.info("processor", "Session captured and registered in pipeline", { session_id: sessionId, path: relativePath });
|
|
3922
5508
|
}
|
|
3923
5509
|
server.registerRoute("POST", "/context", async (req) => {
|
|
3924
5510
|
const { session_id, branch } = ContextBody.parse(req.body);
|
|
@@ -4045,8 +5631,10 @@ ${lines.join("\n")}`;
|
|
|
4045
5631
|
index,
|
|
4046
5632
|
vectorIndex,
|
|
4047
5633
|
llmProvider,
|
|
5634
|
+
digestLlmProvider: digestLlm,
|
|
4048
5635
|
embeddingProvider,
|
|
4049
5636
|
progressTracker,
|
|
5637
|
+
pipeline,
|
|
4050
5638
|
log: (level, message, data) => {
|
|
4051
5639
|
const fn = logger[level];
|
|
4052
5640
|
if (typeof fn === "function") fn.call(logger, "operations", message, data);
|
|
@@ -4057,6 +5645,14 @@ ${lines.join("\n")}`;
|
|
|
4057
5645
|
server.registerRoute("POST", "/api/curate", async (req) => handleCurate(operationDeps, req.body, runCuration));
|
|
4058
5646
|
server.registerRoute("POST", "/api/reprocess", async (req) => handleReprocess(operationDeps, req.body));
|
|
4059
5647
|
server.registerRoute("GET", "/api/sessions", async () => handleGetSessions(index));
|
|
5648
|
+
server.registerRoute("GET", "/api/pipeline/health", handlePipelineHealth(pipeline));
|
|
5649
|
+
server.registerRoute("GET", "/api/pipeline/items", handlePipelineItems(pipeline));
|
|
5650
|
+
server.registerRoute("GET", "/api/pipeline/items/:id", handlePipelineItemDetail(pipeline));
|
|
5651
|
+
server.registerRoute("GET", "/api/pipeline/circuits", handlePipelineCircuits(pipeline));
|
|
5652
|
+
server.registerRoute("POST", "/api/pipeline/retry/:id", handlePipelineRetry(pipeline));
|
|
5653
|
+
server.registerRoute("POST", "/api/pipeline/skip/:id", handlePipelineSkip(pipeline));
|
|
5654
|
+
server.registerRoute("POST", "/api/pipeline/retry-all", handlePipelineRetryAll(pipeline));
|
|
5655
|
+
server.registerRoute("POST", "/api/pipeline/circuit/:provider/reset", handlePipelineCircuitReset(pipeline));
|
|
4060
5656
|
await server.evictExistingDaemon();
|
|
4061
5657
|
const resolvedPort = await resolvePort(config.daemon.port, vaultDir);
|
|
4062
5658
|
if (resolvedPort === 0) {
|
|
@@ -4076,7 +5672,7 @@ ${lines.join("\n")}`;
|
|
|
4076
5672
|
try {
|
|
4077
5673
|
const { loadTemplate } = await import("./templates-XPRBOWCE.js");
|
|
4078
5674
|
const portalContent = loadTemplate("portal", { port: String(server.port) });
|
|
4079
|
-
|
|
5675
|
+
fs8.writeFileSync(path11.join(vaultDir, "_portal.md"), portalContent, "utf-8");
|
|
4080
5676
|
} catch {
|
|
4081
5677
|
}
|
|
4082
5678
|
if (needsMigrationReindex) {
|
|
@@ -4094,6 +5690,8 @@ ${lines.join("\n")}`;
|
|
|
4094
5690
|
await activeStopProcessing;
|
|
4095
5691
|
}
|
|
4096
5692
|
metabolism?.stop();
|
|
5693
|
+
clearInterval(pipelineTickTimer);
|
|
5694
|
+
pipeline.close();
|
|
4097
5695
|
planWatcher.stopFileWatcher();
|
|
4098
5696
|
registry.destroy();
|
|
4099
5697
|
await server.stop();
|
|
@@ -4114,4 +5712,4 @@ export {
|
|
|
4114
5712
|
chokidar/index.js:
|
|
4115
5713
|
(*! chokidar - MIT License (c) 2012 Paul Miller (paulmillr.com) *)
|
|
4116
5714
|
*/
|
|
4117
|
-
//# sourceMappingURL=main-
|
|
5715
|
+
//# sourceMappingURL=main-6AGPIMH2.js.map
|