@mmnto/totem 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunkers/markdown-chunker.d.ts.map +1 -1
- package/dist/chunkers/markdown-chunker.js +9 -4
- package/dist/chunkers/markdown-chunker.js.map +1 -1
- package/dist/chunkers/markdown-chunker.test.js +93 -6
- package/dist/chunkers/markdown-chunker.test.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/ingest/file-resolver.d.ts +6 -1
- package/dist/ingest/file-resolver.d.ts.map +1 -1
- package/dist/ingest/file-resolver.js +50 -7
- package/dist/ingest/file-resolver.js.map +1 -1
- package/dist/ingest/file-resolver.test.d.ts +2 -0
- package/dist/ingest/file-resolver.test.d.ts.map +1 -0
- package/dist/ingest/file-resolver.test.js +84 -0
- package/dist/ingest/file-resolver.test.js.map +1 -0
- package/dist/ingest/pipeline.d.ts.map +1 -1
- package/dist/ingest/pipeline.js +64 -3
- package/dist/ingest/pipeline.js.map +1 -1
- package/dist/ingest/pipeline.test.d.ts +2 -0
- package/dist/ingest/pipeline.test.d.ts.map +1 -0
- package/dist/ingest/pipeline.test.js +79 -0
- package/dist/ingest/pipeline.test.js.map +1 -0
- package/dist/ingest/sync.d.ts +1 -1
- package/dist/ingest/sync.d.ts.map +1 -1
- package/dist/ingest/sync.js +1 -1
- package/dist/ingest/sync.js.map +1 -1
- package/dist/types.d.ts +7 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/xml-format.d.ts +13 -0
- package/dist/xml-format.d.ts.map +1 -0
- package/dist/xml-format.js +17 -0
- package/dist/xml-format.js.map +1 -0
- package/dist/xml-format.test.d.ts +2 -0
- package/dist/xml-format.test.d.ts.map +1 -0
- package/dist/xml-format.test.js +39 -0
- package/dist/xml-format.test.js.map +1 -0
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/markdown-chunker.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACtE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAI5C;;;;;GAKG;AACH,qBAAa,eAAgB,YAAW,OAAO;IAC7C,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAsB;IACtD,OAAO,CAAC,MAAM,CAAsC;gBAExC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI;IAI1C,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,KAAK,EAAE;
|
|
1
|
+
{"version":3,"file":"markdown-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/markdown-chunker.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACtE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAI5C;;;;;GAKG;AACH,qBAAa,eAAgB,YAAW,OAAO;IAC7C,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAsB;IACtD,OAAO,CAAC,MAAM,CAAsC;gBAExC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI;IAI1C,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,KAAK,EAAE;IAwEpE,OAAO,CAAC,kBAAkB;CAuB3B"}
|
|
@@ -20,11 +20,14 @@ export class MarkdownChunker {
|
|
|
20
20
|
const metadata = this.extractFrontmatter(tree, filePath);
|
|
21
21
|
const chunks = [];
|
|
22
22
|
const lines = content.split('\n');
|
|
23
|
-
|
|
23
|
+
// Heading breadcrumb stack: index = depth-1, value = heading text.
|
|
24
|
+
// When we encounter a heading of depth N, we set that slot and truncate deeper levels.
|
|
25
|
+
const breadcrumbs = [];
|
|
24
26
|
let sectionNodes = [];
|
|
25
27
|
let sectionStartLine = 1;
|
|
28
|
+
const getBreadcrumb = () => breadcrumbs.filter(Boolean).join(' > ');
|
|
26
29
|
const flush = () => {
|
|
27
|
-
if (sectionNodes.length === 0 &&
|
|
30
|
+
if (sectionNodes.length === 0 && breadcrumbs.filter(Boolean).length === 0)
|
|
28
31
|
return;
|
|
29
32
|
const sectionText = sectionNodes
|
|
30
33
|
.map((n) => nodeToSourceText(n, lines))
|
|
@@ -32,7 +35,7 @@ export class MarkdownChunker {
|
|
|
32
35
|
.join('\n\n');
|
|
33
36
|
if (!sectionText.trim())
|
|
34
37
|
return;
|
|
35
|
-
const label =
|
|
38
|
+
const label = getBreadcrumb() || filePath;
|
|
36
39
|
const contextPrefix = `File: ${filePath} | Section: ${label}`;
|
|
37
40
|
const endLine = sectionNodes.length > 0
|
|
38
41
|
? (sectionNodes[sectionNodes.length - 1].position?.end.line ?? sectionStartLine)
|
|
@@ -58,7 +61,9 @@ export class MarkdownChunker {
|
|
|
58
61
|
// Only split on headings up to depth 3
|
|
59
62
|
if (h.depth <= MAX_SPLIT_DEPTH) {
|
|
60
63
|
flush();
|
|
61
|
-
|
|
64
|
+
const text = extractPlainText(h.children);
|
|
65
|
+
breadcrumbs[h.depth - 1] = text;
|
|
66
|
+
breadcrumbs.length = h.depth; // truncate deeper levels
|
|
62
67
|
sectionNodes = [];
|
|
63
68
|
sectionStartLine = h.position?.start.line ?? 1;
|
|
64
69
|
continue;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-chunker.js","sourceRoot":"","sources":["../../src/chunkers/markdown-chunker.ts"],"names":[],"mappings":"AACA,OAAO,iBAAiB,MAAM,oBAAoB,CAAC;AACnD,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AAMxB,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B;;;;;GAKG;AACH,MAAM,OAAO,eAAe;IACjB,QAAQ,GAAkB,kBAAkB,CAAC;IAC9C,MAAM,CAAsC;IAEpD,YAAY,MAA8B;QACxC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,QAAgB,EAAE,IAAiB;QACxD,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAS,CAAC;QAEhG,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,
|
|
1
|
+
{"version":3,"file":"markdown-chunker.js","sourceRoot":"","sources":["../../src/chunkers/markdown-chunker.ts"],"names":[],"mappings":"AACA,OAAO,iBAAiB,MAAM,oBAAoB,CAAC;AACnD,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AAMxB,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B;;;;;GAKG;AACH,MAAM,OAAO,eAAe;IACjB,QAAQ,GAAkB,kBAAkB,CAAC;IAC9C,MAAM,CAAsC;IAEpD,YAAY,MAA8B;QACxC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,QAAgB,EAAE,IAAiB;QACxD,MAAM,IAAI,GAAG,OAAO,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAS,CAAC;QAEhG,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,mEAAmE;QACnE,uFAAuF;QACvF,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,MAAM,aAAa,GAAG,GAAW,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE5E,MAAM,KAAK,GAAG,GAAG,EAAE;YACjB,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAElF,MAAM,WAAW,GAAG,YAAY;iBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;iBACtC,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,MAAM,CAAC,CAAC;YAEhB,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;gBAAE,OAAO;YAEhC,MAAM,KAAK,GAAG,aAAa,EAAE,IAAI,QAAQ,CAAC;YAC1C,MAAM,aAAa,GAAG,SAAS,QAAQ,eAAe,KAAK,EAAE,CAAC;YAC9D,MAAM,OAAO,GACX,YAAY,CAAC,MAAM,GAAG,CAAC;gBACrB,CAAC,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,IAAI,gBAAgB,CAAC;gBACjF,CAAC,CAAC,gBAAgB,CAAC;YAEvB,MAAM,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,WAAW,CAAC,IAAI,EAAE;gBAC3B,aAAa;gBACb,QAAQ;gBACR,IAAI;gBACJ,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK;gBACL,SAAS,EAAE,gBAAgB;gBAC3B,OAAO;gBACP,QAAQ;aACT,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjC,6BAA6B;YAC7B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;gBAAE,SAAS;YAEnC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,IAAe,CAAC;gBAC1B,uCAAuC;gBACvC,IAAI,CAAC,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC;oBAC/B,KAAK,EAAE,CAAC;oBACR,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;oBAC1C,WAAW,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;oBAChC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,yBAAyB;oBACvD,YAAY,GAAG,EAAE,CAAC;oBAClB,gBAAgB,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;oBAC/C,SAAS;gBACX,CAAC;YACH,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,sBAAsB;QACtB,KAAK,EAAE,CAAC;QAER,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,IAAU,EAAE,QAAgB;QACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QAC9D,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,IAAI,QAAQ,CAAC;YAAE,OAAO,EAAE,CAAC;QAEnD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAe,CAAC,CAAC;YACpD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;gBAAE,OAAO,EAAE,CAAC;YAE7D,MAAM,MAAM,GAA2B,EAAE,CAAC;YAC1C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9B,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,kCAAkC,QAAQ,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9G,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;YACzC,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,KAAwB;IAChD,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,CAAC,CAAC,KAAK,CAAC;QACtC,IAAI,UAAU,IAAI,CAAC;YAAE,OAAO,gBAAgB,CAAC,CAAC,CAAC,QAA6B,CAAC,CAAC;QAC9E,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAa,EAAE,KAAe;IACtD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;QACtC,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC;IACzE,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
|
@@ -19,11 +19,11 @@ Content of section two.
|
|
|
19
19
|
expect(chunks.length).toBe(3);
|
|
20
20
|
expect(chunks[0].label).toBe('Title');
|
|
21
21
|
expect(chunks[0].content).toContain('Some intro text.');
|
|
22
|
-
expect(chunks[1].label).toBe('Section One');
|
|
22
|
+
expect(chunks[1].label).toBe('Title > Section One');
|
|
23
23
|
expect(chunks[1].content).toContain('Content of section one.');
|
|
24
|
-
expect(chunks[2].label).toBe('Section Two');
|
|
24
|
+
expect(chunks[2].label).toBe('Title > Section Two');
|
|
25
25
|
});
|
|
26
|
-
it('chunks by ### headings', () => {
|
|
26
|
+
it('chunks by ### headings with breadcrumbs', () => {
|
|
27
27
|
const md = `## Parent
|
|
28
28
|
|
|
29
29
|
Parent intro content.
|
|
@@ -40,8 +40,8 @@ Child B content.
|
|
|
40
40
|
expect(chunks.length).toBe(3);
|
|
41
41
|
expect(chunks[0].label).toBe('Parent');
|
|
42
42
|
expect(chunks[0].content).toContain('Parent intro content.');
|
|
43
|
-
expect(chunks[1].label).toBe('Child A');
|
|
44
|
-
expect(chunks[2].label).toBe('Child B');
|
|
43
|
+
expect(chunks[1].label).toBe('Parent > Child A');
|
|
44
|
+
expect(chunks[2].label).toBe('Parent > Child B');
|
|
45
45
|
});
|
|
46
46
|
it('does not split on #### headings', () => {
|
|
47
47
|
const md = `## Section
|
|
@@ -72,7 +72,7 @@ Content here.
|
|
|
72
72
|
date: '2025-10-12',
|
|
73
73
|
});
|
|
74
74
|
});
|
|
75
|
-
it('sets contextPrefix with
|
|
75
|
+
it('sets contextPrefix with breadcrumb trail', () => {
|
|
76
76
|
const md = `## My Section
|
|
77
77
|
|
|
78
78
|
Content.
|
|
@@ -80,6 +80,19 @@ Content.
|
|
|
80
80
|
const chunks = chunker.chunk(md, 'docs/api.md', 'spec');
|
|
81
81
|
expect(chunks[0].contextPrefix).toBe('File: docs/api.md | Section: My Section');
|
|
82
82
|
});
|
|
83
|
+
it('sets contextPrefix with full hierarchy', () => {
|
|
84
|
+
const md = `## Parent
|
|
85
|
+
|
|
86
|
+
Parent content.
|
|
87
|
+
|
|
88
|
+
### Child
|
|
89
|
+
|
|
90
|
+
Child content.
|
|
91
|
+
`;
|
|
92
|
+
const chunks = chunker.chunk(md, 'docs/api.md', 'spec');
|
|
93
|
+
expect(chunks[0].contextPrefix).toBe('File: docs/api.md | Section: Parent');
|
|
94
|
+
expect(chunks[1].contextPrefix).toBe('File: docs/api.md | Section: Parent > Child');
|
|
95
|
+
});
|
|
83
96
|
it('sets correct line numbers', () => {
|
|
84
97
|
const md = `## First
|
|
85
98
|
|
|
@@ -110,5 +123,79 @@ title: Empty
|
|
|
110
123
|
const chunks = chunker.chunk(md, 'test.md', 'session_log');
|
|
111
124
|
expect(chunks[0].type).toBe('session_log');
|
|
112
125
|
});
|
|
126
|
+
// --- Breadcrumb-specific tests ---
|
|
127
|
+
it('resets deeper breadcrumbs when sibling heading appears', () => {
|
|
128
|
+
const md = `## Parent
|
|
129
|
+
|
|
130
|
+
Parent intro.
|
|
131
|
+
|
|
132
|
+
### Child A
|
|
133
|
+
|
|
134
|
+
Child A content.
|
|
135
|
+
|
|
136
|
+
### Child B
|
|
137
|
+
|
|
138
|
+
Child B content.
|
|
139
|
+
|
|
140
|
+
## Other Parent
|
|
141
|
+
|
|
142
|
+
Other content.
|
|
143
|
+
`;
|
|
144
|
+
const chunks = chunker.chunk(md, 'test.md', 'spec');
|
|
145
|
+
expect(chunks[0].label).toBe('Parent');
|
|
146
|
+
expect(chunks[1].label).toBe('Parent > Child A');
|
|
147
|
+
expect(chunks[2].label).toBe('Parent > Child B');
|
|
148
|
+
expect(chunks[3].label).toBe('Other Parent');
|
|
149
|
+
});
|
|
150
|
+
it('handles dangling content before any headings', () => {
|
|
151
|
+
const md = `Some intro text before any headings.
|
|
152
|
+
|
|
153
|
+
## First Section
|
|
154
|
+
|
|
155
|
+
Section content.
|
|
156
|
+
`;
|
|
157
|
+
const chunks = chunker.chunk(md, 'test.md', 'spec');
|
|
158
|
+
expect(chunks.length).toBe(2);
|
|
159
|
+
expect(chunks[0].label).toBe('test.md');
|
|
160
|
+
expect(chunks[0].content).toContain('Some intro text');
|
|
161
|
+
expect(chunks[1].label).toBe('First Section');
|
|
162
|
+
});
|
|
163
|
+
it('handles skipped heading levels (## to ####)', () => {
|
|
164
|
+
const md = `## Top
|
|
165
|
+
|
|
166
|
+
#### Deep (not a split point)
|
|
167
|
+
|
|
168
|
+
Deep content.
|
|
169
|
+
`;
|
|
170
|
+
const chunks = chunker.chunk(md, 'test.md', 'spec');
|
|
171
|
+
// #### doesn't split, so everything stays under ## Top
|
|
172
|
+
expect(chunks.length).toBe(1);
|
|
173
|
+
expect(chunks[0].label).toBe('Top');
|
|
174
|
+
expect(chunks[0].content).toContain('Deep content.');
|
|
175
|
+
});
|
|
176
|
+
it('builds full 3-level breadcrumb trail', () => {
|
|
177
|
+
const md = `# Level 1
|
|
178
|
+
|
|
179
|
+
## Level 2
|
|
180
|
+
|
|
181
|
+
### Level 3
|
|
182
|
+
|
|
183
|
+
Deeply nested content.
|
|
184
|
+
`;
|
|
185
|
+
const chunks = chunker.chunk(md, 'test.md', 'spec');
|
|
186
|
+
const deepChunk = chunks.find((c) => c.content.includes('Deeply nested'));
|
|
187
|
+
expect(deepChunk.label).toBe('Level 1 > Level 2 > Level 3');
|
|
188
|
+
});
|
|
189
|
+
it('correctly builds breadcrumbs for skipped heading levels', () => {
|
|
190
|
+
const md = `# Level 1
|
|
191
|
+
|
|
192
|
+
### Level 3
|
|
193
|
+
|
|
194
|
+
Content for level 3.
|
|
195
|
+
`;
|
|
196
|
+
const chunks = chunker.chunk(md, 'test.md', 'spec');
|
|
197
|
+
const l3Chunk = chunks.find((c) => c.content.includes('Content for level 3'));
|
|
198
|
+
expect(l3Chunk.label).toBe('Level 1 > Level 3');
|
|
199
|
+
});
|
|
113
200
|
});
|
|
114
201
|
//# sourceMappingURL=markdown-chunker.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-chunker.test.js","sourceRoot":"","sources":["../../src/chunkers/markdown-chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAE9C,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,EAAE,GAAG;;;;;;;;;;;CAWd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;QAE3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"markdown-chunker.test.js","sourceRoot":"","sources":["../../src/chunkers/markdown-chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAE9C,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,EAAE,GAAG;;;;;;;;;;;CAWd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;QAE3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QAChE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,EAAE,GAAG;;;;;;;;;;;CAWd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,EAAE,GAAG;;;;;CAKd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,EAAE,GAAG;;;;;;;;CAQd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,aAAa;YACrB,IAAI,EAAE,YAAY;SACnB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,EAAE,GAAG;;;CAGd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;IACnF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,EAAE,GAAG;;;;;;;CAOd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;QAC7E,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,EAAE,GAAG;;;;;;;CAOd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,EAAE,GAAG;;;CAGd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,EAAE,GAAG,wBAAwB,CAAC;QACpC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,oCAAoC;IAEpC,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,EAAE,GAAG;;;;;;;;;;;;;;;CAed,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,EAAE,GAAG;;;;;CAKd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,EAAE,GAAG;;;;;CAKd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,uDAAuD;QACvD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,EAAE,GAAG;;;;;;;CAOd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;QAC1E,MAAM,CAAC,SAAU,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,EAAE,GAAG;;;;;CAKd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC9E,MAAM,CAAC,OAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export type { ChunkStrategy, ContentType, EmbeddingProvider, IngestTarget, Orchestrator, TotemConfig, } from './config-schema.js';
|
|
2
2
|
export { ChunkStrategySchema, ContentTypeSchema, DEFAULT_IGNORE_PATTERNS, EmbeddingProviderSchema, IngestTargetSchema, OllamaProviderSchema, OpenAIProviderSchema, OrchestratorSchema, ShellOrchestratorSchema, TotemConfigSchema, } from './config-schema.js';
|
|
3
|
-
export type { Chunk, SearchOptions, SearchResult, StoredChunk, SyncOptions } from './types.js';
|
|
3
|
+
export type { Chunk, SearchOptions, SearchResult, StoredChunk, SyncOptions, SyncState, } from './types.js';
|
|
4
4
|
export type { Chunker } from './chunkers/chunker.js';
|
|
5
5
|
export { createChunker } from './chunkers/chunker.js';
|
|
6
6
|
export type { Embedder } from './embedders/embedder.js';
|
|
@@ -8,5 +8,6 @@ export { createEmbedder } from './embedders/embedder.js';
|
|
|
8
8
|
export { TOTEM_TABLE_NAME } from './store/lance-schema.js';
|
|
9
9
|
export { LanceStore } from './store/lance-store.js';
|
|
10
10
|
export type { ResolvedFile } from './ingest/sync.js';
|
|
11
|
-
export { getChangedFiles, resolveFiles, runSync } from './ingest/sync.js';
|
|
11
|
+
export { getChangedFiles, getHeadSha, resolveFiles, runSync } from './ingest/sync.js';
|
|
12
|
+
export { wrapXml } from './xml-format.js';
|
|
12
13
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACV,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,YAAY,EACZ,YAAY,EACZ,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EACvB,kBAAkB,EAClB,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAG5B,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACV,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,YAAY,EACZ,YAAY,EACZ,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EACvB,kBAAkB,EAClB,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACV,KAAK,EACL,aAAa,EACb,YAAY,EACZ,WAAW,EACX,WAAW,EACX,SAAS,GACV,MAAM,YAAY,CAAC;AAGpB,YAAY,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAGtD,YAAY,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAGzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAGpD,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAGtF,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -4,5 +4,7 @@ export { createEmbedder } from './embedders/embedder.js';
|
|
|
4
4
|
// Store
|
|
5
5
|
export { TOTEM_TABLE_NAME } from './store/lance-schema.js';
|
|
6
6
|
export { LanceStore } from './store/lance-store.js';
|
|
7
|
-
export { getChangedFiles, resolveFiles, runSync } from './ingest/sync.js';
|
|
7
|
+
export { getChangedFiles, getHeadSha, resolveFiles, runSync } from './ingest/sync.js';
|
|
8
|
+
// Utilities
|
|
9
|
+
export { wrapXml } from './xml-format.js';
|
|
8
10
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AASA,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EACvB,kBAAkB,EAClB,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AASA,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EACvB,kBAAkB,EAClB,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAc5B,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAItD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,QAAQ;AACR,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAIpD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAEtF,YAAY;AACZ,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC"}
|
|
@@ -8,7 +8,12 @@ export interface ResolvedFile {
|
|
|
8
8
|
export declare function resolveFiles(targets: IngestTarget[], projectRoot: string, ignorePatterns?: string[], onWarn?: (msg: string) => void): ResolvedFile[];
|
|
9
9
|
/**
|
|
10
10
|
* Get files changed since a given git ref (e.g., HEAD~1 or a commit SHA).
|
|
11
|
-
*
|
|
11
|
+
* Also includes untracked files so new files are picked up on incremental sync.
|
|
12
|
+
* Uses -z for null-delimited output consistent with getGitNonIgnoredFiles.
|
|
12
13
|
*/
|
|
13
14
|
export declare function getChangedFiles(projectRoot: string, sinceRef?: string, onWarn?: (msg: string) => void): string[] | null;
|
|
15
|
+
/**
|
|
16
|
+
* Get the current HEAD SHA for sync state tracking.
|
|
17
|
+
*/
|
|
18
|
+
export declare function getHeadSha(projectRoot: string, onWarn?: (msg: string) => void): string | null;
|
|
14
19
|
//# sourceMappingURL=file-resolver.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-resolver.d.ts","sourceRoot":"","sources":["../../src/ingest/file-resolver.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGxD,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,YAAY,CAAC;CACtB;AAwCD,sEAAsE;AACtE,wBAAgB,YAAY,CAC1B,OAAO,EAAE,YAAY,EAAE,EACvB,WAAW,EAAE,MAAM,EACnB,cAAc,GAAE,MAAM,EAA4B,EAClD,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAC7B,YAAY,EAAE,CA2BhB;
|
|
1
|
+
{"version":3,"file":"file-resolver.d.ts","sourceRoot":"","sources":["../../src/ingest/file-resolver.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGxD,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,YAAY,CAAC;CACtB;AAwCD,sEAAsE;AACtE,wBAAgB,YAAY,CAC1B,OAAO,EAAE,YAAY,EAAE,EACvB,WAAW,EAAE,MAAM,EACnB,cAAc,GAAE,MAAM,EAA4B,EAClD,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAC7B,YAAY,EAAE,CA2BhB;AAKD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,QAAQ,GAAE,MAAiB,EAC3B,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAC7B,MAAM,EAAE,GAAG,IAAI,CA8CjB;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAAG,MAAM,GAAG,IAAI,CAa7F"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { execFileSync
|
|
1
|
+
import { execFileSync } from 'node:child_process';
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import { globSync } from 'glob';
|
|
4
4
|
import { DEFAULT_IGNORE_PATTERNS } from '../config-schema.js';
|
|
@@ -57,20 +57,45 @@ export function resolveFiles(targets, projectRoot, ignorePatterns = DEFAULT_IGNO
|
|
|
57
57
|
}
|
|
58
58
|
return results;
|
|
59
59
|
}
|
|
60
|
+
/** Validate that a git ref is safe (hex SHA, HEAD~N, branch name — no shell metacharacters). */
|
|
61
|
+
const SAFE_GIT_REF = /^[a-zA-Z0-9_./:~^{}\-]+$/;
|
|
60
62
|
/**
|
|
61
63
|
* Get files changed since a given git ref (e.g., HEAD~1 or a commit SHA).
|
|
62
|
-
*
|
|
64
|
+
* Also includes untracked files so new files are picked up on incremental sync.
|
|
65
|
+
* Uses -z for null-delimited output consistent with getGitNonIgnoredFiles.
|
|
63
66
|
*/
|
|
64
67
|
export function getChangedFiles(projectRoot, sinceRef = 'HEAD~1', onWarn) {
|
|
68
|
+
if (!SAFE_GIT_REF.test(sinceRef)) {
|
|
69
|
+
if (onWarn) {
|
|
70
|
+
onWarn(`Invalid git ref "${sinceRef}" — falling back to full sync.`);
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
65
74
|
try {
|
|
66
|
-
const
|
|
75
|
+
const diffOutput = execFileSync('git', ['diff', '-z', '--name-only', sinceRef], {
|
|
67
76
|
cwd: projectRoot,
|
|
68
77
|
encoding: 'utf-8',
|
|
78
|
+
shell: process.platform === 'win32',
|
|
69
79
|
});
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
80
|
+
// Also pick up untracked files (new files not yet committed)
|
|
81
|
+
let untrackedOutput = '';
|
|
82
|
+
try {
|
|
83
|
+
untrackedOutput = execFileSync('git', ['ls-files', '-z', '--others', '--exclude-standard'], {
|
|
84
|
+
cwd: projectRoot,
|
|
85
|
+
encoding: 'utf-8',
|
|
86
|
+
shell: process.platform === 'win32',
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
if (onWarn) {
|
|
91
|
+
onWarn(`Failed to list untracked files: ${err instanceof Error ? err.message : String(err)}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
const paths = new Set((diffOutput + untrackedOutput)
|
|
95
|
+
.split('\0')
|
|
96
|
+
.map((p) => p.replace(/\\/g, '/'))
|
|
97
|
+
.filter(Boolean));
|
|
98
|
+
return [...paths];
|
|
74
99
|
}
|
|
75
100
|
catch (err) {
|
|
76
101
|
const msg = `Failed to get changed files from git. Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
@@ -80,4 +105,22 @@ export function getChangedFiles(projectRoot, sinceRef = 'HEAD~1', onWarn) {
|
|
|
80
105
|
return null;
|
|
81
106
|
}
|
|
82
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* Get the current HEAD SHA for sync state tracking.
|
|
110
|
+
*/
|
|
111
|
+
export function getHeadSha(projectRoot, onWarn) {
|
|
112
|
+
try {
|
|
113
|
+
return execFileSync('git', ['rev-parse', 'HEAD'], {
|
|
114
|
+
cwd: projectRoot,
|
|
115
|
+
encoding: 'utf-8',
|
|
116
|
+
shell: process.platform === 'win32',
|
|
117
|
+
}).trim();
|
|
118
|
+
}
|
|
119
|
+
catch (err) {
|
|
120
|
+
if (onWarn) {
|
|
121
|
+
onWarn(`Failed to read HEAD SHA: ${err instanceof Error ? err.message : String(err)}`);
|
|
122
|
+
}
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
83
126
|
//# sourceMappingURL=file-resolver.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-resolver.js","sourceRoot":"","sources":["../../src/ingest/file-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"file-resolver.js","sourceRoot":"","sources":["../../src/ingest/file-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAGhC,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAQ9D;;;GAGG;AACH,SAAS,qBAAqB,CAC5B,WAAmB,EACnB,MAA8B;IAE9B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,YAAY,CACzB,KAAK,EACL,CAAC,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,oBAAoB,CAAC,EAChE;YACE,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,OAAO;YACjB,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;YAC3B,KAAK,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO;SACpC,CACF,CAAC;QACF,OAAO,IAAI,GAAG,CACZ,MAAM;aACH,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,OAAO,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CACrC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClE,MAAM,GAAG,GACP,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;YAC3D,CAAC,CAAC,oGAAoG;YACtG,CAAC,CAAC,kGAAkG,QAAQ,EAAE,CAAC;QACnH,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,YAAY,CAC1B,OAAuB,EACvB,WAAmB,EACnB,iBAA2B,uBAAuB,EAClD,MAA8B;IAE9B,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,UAAU,GAAG,qBAAqB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAE9D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE;YACpC,GAAG,EAAE,WAAW;YAChB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,cAAc;SACvB,CAAC,CAAC;QAEH,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACjD,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC;gBAAE,SAAS;YACrC,IAAI,UAAU,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC;gBAAE,SAAS;YAC1D,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YAEvB,OAAO,CAAC,IAAI,CAAC;gBACX,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC;gBAC7C,YAAY;gBACZ,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,gGAAgG;AAChG,MAAM,YAAY,GAAG,0BAA0B,CAAC;AAEhD;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,WAAmB,QAAQ,EAC3B,MAA8B;IAE9B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjC,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,oBAAoB,QAAQ,gCAAgC,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE;YAC9E,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO;SACpC,CAAC,CAAC;QAEH,6DAA6D;QAC7D,IAAI,eAAe,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,eAAe,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,oBAAoB,CAAC,EAAE;gBAC1F,GAAG,EAAE,WAAW;gBAChB,QAAQ,EAAE,OAAO;gBACjB,KAAK,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,CACJ,mCAAmC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,CAAC,UAAU,GAAG,eAAe,CAAC;aAC3B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;aACjC,MAAM,CAAC,OAAO,CAAC,CACnB,CAAC;QAEF,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;IACpB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,gDAAgD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/G,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,WAAmB,EAAE,MAA8B;IAC5E,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE;YAChD,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO;SACpC,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,4BAA4B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-resolver.test.d.ts","sourceRoot":"","sources":["../../src/ingest/file-resolver.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import * as childProcess from 'node:child_process';
|
|
2
|
+
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
3
|
+
import { getChangedFiles, getHeadSha } from './file-resolver.js';
|
|
4
|
+
vi.mock('node:child_process', () => ({
|
|
5
|
+
execFileSync: vi.fn(),
|
|
6
|
+
}));
|
|
7
|
+
afterEach(() => {
|
|
8
|
+
vi.restoreAllMocks();
|
|
9
|
+
});
|
|
10
|
+
describe('getHeadSha', () => {
|
|
11
|
+
it('returns trimmed SHA on success', () => {
|
|
12
|
+
vi.mocked(childProcess.execFileSync).mockReturnValue('abc123def456\n');
|
|
13
|
+
expect(getHeadSha('/project')).toBe('abc123def456');
|
|
14
|
+
});
|
|
15
|
+
it('returns null when git fails', () => {
|
|
16
|
+
vi.mocked(childProcess.execFileSync).mockImplementation(() => {
|
|
17
|
+
throw new Error('not a git repo');
|
|
18
|
+
});
|
|
19
|
+
expect(getHeadSha('/project')).toBeNull();
|
|
20
|
+
});
|
|
21
|
+
it('calls onWarn when git fails', () => {
|
|
22
|
+
vi.mocked(childProcess.execFileSync).mockImplementation(() => {
|
|
23
|
+
throw new Error('not a git repo');
|
|
24
|
+
});
|
|
25
|
+
const warn = vi.fn();
|
|
26
|
+
getHeadSha('/project', warn);
|
|
27
|
+
expect(warn).toHaveBeenCalledWith(expect.stringContaining('not a git repo'));
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
describe('getChangedFiles', () => {
|
|
31
|
+
it('returns deduplicated paths from diff and untracked (null-delimited)', () => {
|
|
32
|
+
vi.mocked(childProcess.execFileSync).mockImplementation((_cmd, args) => {
|
|
33
|
+
if (args && args.includes('diff'))
|
|
34
|
+
return 'src/a.ts\0src/b.ts\0';
|
|
35
|
+
if (args && args.includes('ls-files'))
|
|
36
|
+
return 'src/b.ts\0src/new.ts\0';
|
|
37
|
+
return '';
|
|
38
|
+
});
|
|
39
|
+
const result = getChangedFiles('/project', 'HEAD~1');
|
|
40
|
+
expect(result).toEqual(expect.arrayContaining(['src/a.ts', 'src/b.ts', 'src/new.ts']));
|
|
41
|
+
expect(result).toHaveLength(3);
|
|
42
|
+
});
|
|
43
|
+
it('returns null and warns when git diff fails', () => {
|
|
44
|
+
vi.mocked(childProcess.execFileSync).mockImplementation(() => {
|
|
45
|
+
throw new Error('bad ref');
|
|
46
|
+
});
|
|
47
|
+
const warn = vi.fn();
|
|
48
|
+
const result = getChangedFiles('/project', 'HEAD~1', warn);
|
|
49
|
+
expect(result).toBeNull();
|
|
50
|
+
expect(warn).toHaveBeenCalledWith(expect.stringContaining('bad ref'));
|
|
51
|
+
});
|
|
52
|
+
it('still returns diff results when untracked listing fails', () => {
|
|
53
|
+
vi.mocked(childProcess.execFileSync).mockImplementation((_cmd, args) => {
|
|
54
|
+
if (args && args.includes('diff'))
|
|
55
|
+
return 'src/a.ts\0';
|
|
56
|
+
throw new Error('ls-files failed');
|
|
57
|
+
});
|
|
58
|
+
const warn = vi.fn();
|
|
59
|
+
const result = getChangedFiles('/project', 'HEAD~1', warn);
|
|
60
|
+
expect(result).toEqual(['src/a.ts']);
|
|
61
|
+
expect(warn).toHaveBeenCalledWith(expect.stringContaining('untracked'));
|
|
62
|
+
});
|
|
63
|
+
it('normalizes backslashes to forward slashes', () => {
|
|
64
|
+
vi.mocked(childProcess.execFileSync).mockImplementation((_cmd, args) => {
|
|
65
|
+
if (args && args.includes('diff'))
|
|
66
|
+
return 'src\\foo\\bar.ts\0';
|
|
67
|
+
return '';
|
|
68
|
+
});
|
|
69
|
+
const result = getChangedFiles('/project');
|
|
70
|
+
expect(result).toEqual(['src/foo/bar.ts']);
|
|
71
|
+
});
|
|
72
|
+
it('rejects sinceRef with shell metacharacters', () => {
|
|
73
|
+
const warn = vi.fn();
|
|
74
|
+
const result = getChangedFiles('/project', 'HEAD; rm -rf /', warn);
|
|
75
|
+
expect(result).toBeNull();
|
|
76
|
+
expect(warn).toHaveBeenCalledWith(expect.stringContaining('Invalid git ref'));
|
|
77
|
+
});
|
|
78
|
+
it('accepts valid hex SHA as sinceRef', () => {
|
|
79
|
+
vi.mocked(childProcess.execFileSync).mockReturnValue('');
|
|
80
|
+
const result = getChangedFiles('/project', 'abc123def456');
|
|
81
|
+
expect(result).toEqual([]);
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
//# sourceMappingURL=file-resolver.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-resolver.test.js","sourceRoot":"","sources":["../../src/ingest/file-resolver.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,oBAAoB,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE7D,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEjE,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;CACtB,CAAC,CAAC,CAAC;AAEJ,SAAS,CAAC,GAAG,EAAE;IACb,EAAE,CAAC,eAAe,EAAE,CAAC;AACvB,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;QACvE,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QACrB,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,qEAAqE,EAAE,GAAG,EAAE;QAC7E,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,kBAAkB,CACrD,CAAC,IAAY,EAAE,IAAwB,EAAE,EAAE;YACzC,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,sBAAsB,CAAC;YACjE,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,OAAO,wBAAwB,CAAC;YACvE,OAAO,EAAE,CAAC;QACZ,CAAC,CACF,CAAC;QACF,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QACvF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,kBAAkB,CACrD,CAAC,IAAY,EAAE,IAAwB,EAAE,EAAE;YACzC,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,YAAY,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC,CACF,CAAC;QACF,MAAM,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,kBAAkB,CACrD,CAAC,IAAY,EAAE,IAAwB,EAAE,EAAE;YACzC,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,oBAAoB,CAAC;YAC/D,OAAO,EAAE,CAAC;QACZ,CAAC,CACF,CAAC;QACF,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;QACnE,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvD,OAAO,KAAK,EAAS,WAAW,
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvD,OAAO,KAAK,EAAS,WAAW,EAAa,MAAM,aAAa,CAAC;AAwCjE,wBAAsB,OAAO,CAC3B,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,CAAC,CA4I9D"}
|
package/dist/ingest/pipeline.js
CHANGED
|
@@ -3,12 +3,43 @@ import * as path from 'node:path';
|
|
|
3
3
|
import { createChunker } from '../chunkers/chunker.js';
|
|
4
4
|
import { createEmbedder } from '../embedders/embedder.js';
|
|
5
5
|
import { LanceStore } from '../store/lance-store.js';
|
|
6
|
-
import { getChangedFiles, resolveFiles } from './file-resolver.js';
|
|
6
|
+
import { getChangedFiles, getHeadSha, resolveFiles } from './file-resolver.js';
|
|
7
7
|
const EMBED_BATCH_SIZE = 100;
|
|
8
|
+
const SYNC_STATE_FILE = 'cache/sync-state.json';
|
|
9
|
+
function readSyncState(totemDir, onProgress) {
|
|
10
|
+
const statePath = path.join(totemDir, SYNC_STATE_FILE);
|
|
11
|
+
try {
|
|
12
|
+
const raw = fs.readFileSync(statePath, 'utf-8');
|
|
13
|
+
const parsed = JSON.parse(raw);
|
|
14
|
+
if (typeof parsed.lastSyncSha === 'string' &&
|
|
15
|
+
parsed.lastSyncSha &&
|
|
16
|
+
typeof parsed.timestamp === 'number') {
|
|
17
|
+
return parsed;
|
|
18
|
+
}
|
|
19
|
+
onProgress?.(`Warning: Ignoring malformed sync state file at ${statePath}.`);
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
catch (err) {
|
|
23
|
+
// ENOENT is expected on first run — don't warn
|
|
24
|
+
if (err instanceof Error && err.code === 'ENOENT') {
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
onProgress?.(`Warning: Failed to read sync state: ${err instanceof Error ? err.message : String(err)}`);
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function writeSyncState(totemDir, state) {
|
|
32
|
+
const statePath = path.join(totemDir, SYNC_STATE_FILE);
|
|
33
|
+
const dir = path.dirname(statePath);
|
|
34
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
35
|
+
fs.writeFileSync(statePath, JSON.stringify(state, null, 2) + '\n', 'utf-8');
|
|
36
|
+
}
|
|
8
37
|
export async function runSync(config, options) {
|
|
9
38
|
const { projectRoot, onProgress } = options;
|
|
10
39
|
let incremental = options.incremental;
|
|
11
40
|
const log = onProgress ?? (() => { });
|
|
41
|
+
// 0. Capture HEAD SHA early — before any async work that might race with new commits
|
|
42
|
+
const headSha = getHeadSha(projectRoot, log);
|
|
12
43
|
// 1. Create embedder
|
|
13
44
|
log('Initializing embedding provider...');
|
|
14
45
|
const embedder = createEmbedder(config.embedding);
|
|
@@ -22,10 +53,21 @@ export async function runSync(config, options) {
|
|
|
22
53
|
incremental = false;
|
|
23
54
|
}
|
|
24
55
|
// 3. Resolve files to process
|
|
56
|
+
const totemDir = path.join(projectRoot, config.totemDir);
|
|
25
57
|
const allFiles = resolveFiles(config.targets, projectRoot, config.ignorePatterns, log);
|
|
26
58
|
let filesToProcess;
|
|
59
|
+
let deletedPaths = [];
|
|
27
60
|
if (incremental) {
|
|
28
|
-
|
|
61
|
+
// Determine the ref to diff against: saved sync state > fallback HEAD~1
|
|
62
|
+
let sinceRef = 'HEAD~1';
|
|
63
|
+
if (!options.changedFiles) {
|
|
64
|
+
const syncState = readSyncState(totemDir, log);
|
|
65
|
+
if (syncState) {
|
|
66
|
+
sinceRef = syncState.lastSyncSha;
|
|
67
|
+
log(`Resuming from last sync at ${sinceRef.slice(0, 8)}...`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const changedPaths = options.changedFiles ?? getChangedFiles(projectRoot, sinceRef, log);
|
|
29
71
|
if (changedPaths === null) {
|
|
30
72
|
log('Git diff failed, falling back to full sync...');
|
|
31
73
|
await store.reset();
|
|
@@ -34,8 +76,13 @@ export async function runSync(config, options) {
|
|
|
34
76
|
}
|
|
35
77
|
else {
|
|
36
78
|
const changedSet = new Set(changedPaths);
|
|
79
|
+
const allFileSet = new Set(allFiles.map((f) => f.relativePath));
|
|
80
|
+
// Partition: files that still exist get re-indexed, missing files get deleted
|
|
37
81
|
filesToProcess = allFiles.filter((f) => changedSet.has(f.relativePath));
|
|
38
|
-
|
|
82
|
+
deletedPaths = changedPaths.filter((p) => !allFileSet.has(p));
|
|
83
|
+
log(`Incremental sync: ${filesToProcess.length} changed files` +
|
|
84
|
+
(deletedPaths.length > 0 ? `, ${deletedPaths.length} deleted` : '') +
|
|
85
|
+
` (of ${allFiles.length} total)`);
|
|
39
86
|
}
|
|
40
87
|
}
|
|
41
88
|
else {
|
|
@@ -44,6 +91,16 @@ export async function runSync(config, options) {
|
|
|
44
91
|
filesToProcess = allFiles;
|
|
45
92
|
log(`Full sync: ${filesToProcess.length} files to process`);
|
|
46
93
|
}
|
|
94
|
+
// 3b. Purge chunks from deleted files before ingesting new ones
|
|
95
|
+
for (const deletedPath of deletedPaths) {
|
|
96
|
+
try {
|
|
97
|
+
await store.deleteByFile(deletedPath);
|
|
98
|
+
log(` Purged chunks for deleted file: ${deletedPath}`);
|
|
99
|
+
}
|
|
100
|
+
catch (err) {
|
|
101
|
+
log(` Warning: failed to purge ${deletedPath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
47
104
|
// 4. Chunk files and stream to LanceDB in batches (bounded memory)
|
|
48
105
|
let totalChunks = 0;
|
|
49
106
|
let buffer = [];
|
|
@@ -85,6 +142,10 @@ export async function runSync(config, options) {
|
|
|
85
142
|
// Flush remaining chunks
|
|
86
143
|
await flushBuffer();
|
|
87
144
|
log(`Sync complete: ${totalChunks} chunks from ${filesToProcess.length} files`);
|
|
145
|
+
// Persist sync state so next incremental sync knows where to diff from
|
|
146
|
+
if (headSha) {
|
|
147
|
+
writeSyncState(totemDir, { lastSyncSha: headSha, timestamp: Date.now() });
|
|
148
|
+
}
|
|
88
149
|
return {
|
|
89
150
|
chunksProcessed: totalChunks,
|
|
90
151
|
filesProcessed: filesToProcess.length,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE/E,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B,MAAM,eAAe,GAAG,uBAAuB,CAAC;AAEhD,SAAS,aAAa,CAAC,QAAgB,EAAE,UAAkC;IACzE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAc,CAAC;QAC5C,IACE,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ;YACtC,MAAM,CAAC,WAAW;YAClB,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,EACpC,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,UAAU,EAAE,CAAC,kDAAkD,SAAS,GAAG,CAAC,CAAC;QAC7E,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,+CAA+C;QAC/C,IAAI,GAAG,YAAY,KAAK,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,UAAU,EAAE,CACV,uCAAuC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC1F,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB,EAAE,KAAgB;IACxD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAC9E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,MAAmB,EACnB,OAAoB;IAEpB,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAC5C,IAAI,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IACtC,MAAM,GAAG,GAAG,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAErC,qFAAqF;IACrF,MAAM,OAAO,GAAG,UAAU,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IAE7C,qBAAqB;IACrB,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAElD,sBAAsB;IACtB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAClD,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IAEtB,+EAA+E;IAC/E,IAAI,WAAW,IAAI,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QAC3C,GAAG,CAAC,+CAA+C,CAAC,CAAC;QACrD,WAAW,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACvF,IAAI,cAA8B,CAAC;IACnC,IAAI,YAAY,GAAa,EAAE,CAAC;IAEhC,IAAI,WAAW,EAAE,CAAC;QAChB,wEAAwE;QACxE,IAAI,QAAQ,GAAG,QAAQ,CAAC;QACxB,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;YAC1B,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAC/C,IAAI,SAAS,EAAE,CAAC;gBACd,QAAQ,GAAG,SAAS,CAAC,WAAW,CAAC;gBACjC,GAAG,CAAC,8BAA8B,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,eAAe,CAAC,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;QACzF,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,GAAG,CAAC,+CAA+C,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,cAAc,GAAG,QAAQ,CAAC;YAC1B,GAAG,CAAC,yBAAyB,cAAc,CAAC,MAAM,mBAAmB,CAAC,CAAC;QACzE,CAAC;aAAM,CAAC;YACN,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;YACzC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;YAEhE,8EAA8E;YAC9E,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;YACxE,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAE9D,GAAG,CACD,qBAAqB,cAAc,CAAC,MAAM,gBAAgB;gBACxD,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnE,QAAQ,QAAQ,CAAC,MAAM,SAAS,CACnC,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,+BAA+B,CAAC,CAAC;QACrC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,cAAc,GAAG,QAAQ,CAAC;QAC1B,GAAG,CAAC,cAAc,cAAc,CAAC,MAAM,mBAAmB,CAAC,CAAC;IAC9D,CAAC;IAED,gEAAgE;IAChE,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;YACtC,GAAG,CAAC,qCAAqC,WAAW,EAAE,CAAC,CAAC;QAC1D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CACD,8BAA8B,WAAW,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACjG,CAAC;QACJ,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,MAAM,GAAY,EAAE,CAAC;IAEzB,KAAK,UAAU,WAAW;QACxB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAChC,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3B,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;QAC7B,GAAG,CAAC,cAAc,WAAW,gBAAgB,CAAC,CAAC;QAC/C,MAAM,GAAG,EAAE,CAAC;IACd,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,GAAG,CAAC,YAAY,IAAI,CAAC,YAAY,KAAK,CAAC,CAAC;QAExC,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CACD,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CACrG,CAAC;YACF,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAE3E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,GAAG,CAAC,8BAA8B,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;YACvD,SAAS;QACX,CAAC;QAED,6EAA6E;QAC7E,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QACvB,GAAG,CAAC,KAAK,MAAM,CAAC,MAAM,gBAAgB,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAE3D,8DAA8D;QAC9D,IAAI,MAAM,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACtC,MAAM,WAAW,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,WAAW,EAAE,CAAC;IAEpB,GAAG,CAAC,kBAAkB,WAAW,gBAAgB,cAAc,CAAC,MAAM,QAAQ,CAAC,CAAC;IAEhF,uEAAuE;IACvE,IAAI,OAAO,EAAE,CAAC;QACZ,cAAc,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO;QACL,eAAe,EAAE,WAAW;QAC5B,cAAc,EAAE,cAAc,CAAC,MAAM;KACtC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.test.d.ts","sourceRoot":"","sources":["../../src/ingest/pipeline.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as os from 'node:os';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
5
|
+
// Import the internal helpers via a workaround — we test the state file contract
|
|
6
|
+
// since readSyncState/writeSyncState are not exported directly.
|
|
7
|
+
const SYNC_STATE_FILE = 'cache/sync-state.json';
|
|
8
|
+
describe('sync state persistence', () => {
|
|
9
|
+
let tmpDir;
|
|
10
|
+
beforeEach(() => {
|
|
11
|
+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'totem-sync-state-'));
|
|
12
|
+
});
|
|
13
|
+
afterEach(() => {
|
|
14
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
15
|
+
});
|
|
16
|
+
it('creates cache directory and writes valid JSON', () => {
|
|
17
|
+
const statePath = path.join(tmpDir, SYNC_STATE_FILE);
|
|
18
|
+
const dir = path.dirname(statePath);
|
|
19
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
20
|
+
const state = { lastSyncSha: 'abc123def456', timestamp: Date.now() };
|
|
21
|
+
fs.writeFileSync(statePath, JSON.stringify(state, null, 2) + '\n', 'utf-8');
|
|
22
|
+
const raw = fs.readFileSync(statePath, 'utf-8');
|
|
23
|
+
const parsed = JSON.parse(raw);
|
|
24
|
+
expect(parsed.lastSyncSha).toBe('abc123def456');
|
|
25
|
+
expect(typeof parsed.timestamp).toBe('number');
|
|
26
|
+
});
|
|
27
|
+
it('reads back a previously written state', () => {
|
|
28
|
+
const statePath = path.join(tmpDir, SYNC_STATE_FILE);
|
|
29
|
+
const dir = path.dirname(statePath);
|
|
30
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
31
|
+
const state = { lastSyncSha: 'deadbeef', timestamp: 1234567890 };
|
|
32
|
+
fs.writeFileSync(statePath, JSON.stringify(state), 'utf-8');
|
|
33
|
+
const raw = fs.readFileSync(statePath, 'utf-8');
|
|
34
|
+
const parsed = JSON.parse(raw);
|
|
35
|
+
expect(parsed.lastSyncSha).toBe('deadbeef');
|
|
36
|
+
expect(parsed.timestamp).toBe(1234567890);
|
|
37
|
+
});
|
|
38
|
+
it('returns null-equivalent for missing state file', () => {
|
|
39
|
+
const statePath = path.join(tmpDir, SYNC_STATE_FILE);
|
|
40
|
+
expect(fs.existsSync(statePath)).toBe(false);
|
|
41
|
+
});
|
|
42
|
+
it('returns null-equivalent for corrupted state file', () => {
|
|
43
|
+
const statePath = path.join(tmpDir, SYNC_STATE_FILE);
|
|
44
|
+
const dir = path.dirname(statePath);
|
|
45
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
46
|
+
fs.writeFileSync(statePath, '{ broken json!!!', 'utf-8');
|
|
47
|
+
let parsed = null;
|
|
48
|
+
try {
|
|
49
|
+
parsed = JSON.parse(fs.readFileSync(statePath, 'utf-8'));
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
// Expected
|
|
53
|
+
}
|
|
54
|
+
expect(parsed).toBeNull();
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
describe('deleted file partitioning', () => {
|
|
58
|
+
it('identifies files in changedPaths but missing from allFiles as deleted', () => {
|
|
59
|
+
const changedPaths = ['src/a.ts', 'src/b.ts', 'src/deleted.ts'];
|
|
60
|
+
const allFileSet = new Set(['src/a.ts', 'src/b.ts']);
|
|
61
|
+
const deletedPaths = changedPaths.filter((p) => !allFileSet.has(p));
|
|
62
|
+
expect(deletedPaths).toEqual(['src/deleted.ts']);
|
|
63
|
+
});
|
|
64
|
+
it('returns empty when all changed files still exist', () => {
|
|
65
|
+
const changedPaths = ['src/a.ts'];
|
|
66
|
+
const allFileSet = new Set(['src/a.ts', 'src/b.ts']);
|
|
67
|
+
const deletedPaths = changedPaths.filter((p) => !allFileSet.has(p));
|
|
68
|
+
expect(deletedPaths).toEqual([]);
|
|
69
|
+
});
|
|
70
|
+
it('handles renamed files (old path deleted, new path exists)', () => {
|
|
71
|
+
const changedPaths = ['src/old-name.ts', 'src/new-name.ts'];
|
|
72
|
+
const allFileSet = new Set(['src/new-name.ts']);
|
|
73
|
+
const filesToProcess = changedPaths.filter((p) => allFileSet.has(p));
|
|
74
|
+
const deletedPaths = changedPaths.filter((p) => !allFileSet.has(p));
|
|
75
|
+
expect(filesToProcess).toEqual(['src/new-name.ts']);
|
|
76
|
+
expect(deletedPaths).toEqual(['src/old-name.ts']);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
//# sourceMappingURL=pipeline.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.test.js","sourceRoot":"","sources":["../../src/ingest/pipeline.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAErE,iFAAiF;AACjF,gEAAgE;AAEhE,MAAM,eAAe,GAAG,uBAAuB,CAAC;AAEhD,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,IAAI,MAAc,CAAC;IAEnB,UAAU,CAAC,GAAG,EAAE;QACd,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvC,MAAM,KAAK,GAAG,EAAE,WAAW,EAAE,cAAc,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACrE,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;QAE5E,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChD,MAAM,CAAC,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvC,MAAM,KAAK,GAAG,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;QACjE,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;QAE5D,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACrD,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC;QAEzD,IAAI,MAAM,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,WAAW;QACb,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,uEAAuE,EAAE,GAAG,EAAE;QAC/E,MAAM,YAAY,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAC;QAChE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;QAErD,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpE,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,YAAY,GAAG,CAAC,UAAU,CAAC,CAAC;QAClC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;QAErD,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpE,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,MAAM,YAAY,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;QAC5D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpE,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC;QACpD,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/ingest/sync.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export type { ResolvedFile } from './file-resolver.js';
|
|
2
|
-
export { getChangedFiles, resolveFiles } from './file-resolver.js';
|
|
2
|
+
export { getChangedFiles, getHeadSha, resolveFiles } from './file-resolver.js';
|
|
3
3
|
export { runSync } from './pipeline.js';
|
|
4
4
|
//# sourceMappingURL=sync.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/ingest/sync.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/ingest/sync.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC"}
|
package/dist/ingest/sync.js
CHANGED
package/dist/ingest/sync.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/ingest/sync.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/ingest/sync.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -64,6 +64,13 @@ export interface SyncOptions {
|
|
|
64
64
|
/** Callback for progress reporting */
|
|
65
65
|
onProgress?: (message: string) => void;
|
|
66
66
|
}
|
|
67
|
+
/**
|
|
68
|
+
* Persisted state for incremental sync tracking.
|
|
69
|
+
*/
|
|
70
|
+
export interface SyncState {
|
|
71
|
+
lastSyncSha: string;
|
|
72
|
+
timestamp: number;
|
|
73
|
+
}
|
|
67
74
|
/**
|
|
68
75
|
* Options for search queries.
|
|
69
76
|
*/
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAErE;;;GAGG;AACH,MAAM,WAAW,KAAK;IACpB,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAC;IAEhB,mDAAmD;IACnD,aAAa,EAAE,MAAM,CAAC;IAEtB,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IAEjB,0CAA0C;IAC1C,IAAI,EAAE,WAAW,CAAC;IAElB,iDAAiD;IACjD,QAAQ,EAAE,aAAa,CAAC;IAExB,oEAAoE;IACpE,KAAK,EAAE,MAAM,CAAC;IAEd,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;IAElB,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;IAEhB,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;IAEpB,0DAA0D;IAC1D,WAAW,EAAE,OAAO,CAAC;IAErB,gEAAgE;IAChE,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IAExB,sCAAsC;IACtC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,WAAW,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAErE;;;GAGG;AACH,MAAM,WAAW,KAAK;IACpB,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAC;IAEhB,mDAAmD;IACnD,aAAa,EAAE,MAAM,CAAC;IAEtB,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IAEjB,0CAA0C;IAC1C,IAAI,EAAE,WAAW,CAAC;IAElB,iDAAiD;IACjD,QAAQ,EAAE,aAAa,CAAC;IAExB,oEAAoE;IACpE,KAAK,EAAE,MAAM,CAAC;IAEd,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;IAElB,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;IAEhB,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;IAEpB,0DAA0D;IAC1D,WAAW,EAAE,OAAO,CAAC;IAErB,gEAAgE;IAChE,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IAExB,sCAAsC;IACtC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,WAAW,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wrap external/user-supplied content in XML tags to create a clear boundary
|
|
3
|
+
* between system instructions and passive data. Escapes matching closing tags
|
|
4
|
+
* in the content using backslash escaping to prevent breakout.
|
|
5
|
+
*
|
|
6
|
+
* Backslash escaping is preferred over HTML entities for LLM consumption —
|
|
7
|
+
* it prevents tag interpretation while maintaining readability.
|
|
8
|
+
*
|
|
9
|
+
* @see https://github.com/mmnto-ai/totem/issues/149
|
|
10
|
+
* @see https://github.com/mmnto-ai/totem/issues/158
|
|
11
|
+
*/
|
|
12
|
+
export declare function wrapXml(tag: string, content: string): string;
|
|
13
|
+
//# sourceMappingURL=xml-format.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xml-format.d.ts","sourceRoot":"","sources":["../src/xml-format.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAO5D"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wrap external/user-supplied content in XML tags to create a clear boundary
|
|
3
|
+
* between system instructions and passive data. Escapes matching closing tags
|
|
4
|
+
* in the content using backslash escaping to prevent breakout.
|
|
5
|
+
*
|
|
6
|
+
* Backslash escaping is preferred over HTML entities for LLM consumption —
|
|
7
|
+
* it prevents tag interpretation while maintaining readability.
|
|
8
|
+
*
|
|
9
|
+
* @see https://github.com/mmnto-ai/totem/issues/149
|
|
10
|
+
* @see https://github.com/mmnto-ai/totem/issues/158
|
|
11
|
+
*/
|
|
12
|
+
export function wrapXml(tag, content) {
|
|
13
|
+
const safeTag = tag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
14
|
+
const escaped = content.replace(new RegExp(`</\\s*${safeTag}\\s*>`, 'gi'), (match) => `<\\/${match.slice(2)}`);
|
|
15
|
+
return `<${tag}>\n${escaped}\n</${tag}>`;
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=xml-format.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xml-format.js","sourceRoot":"","sources":["../src/xml-format.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,MAAM,UAAU,OAAO,CAAC,GAAW,EAAE,OAAe;IAClD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAC7B,IAAI,MAAM,CAAC,SAAS,OAAO,OAAO,EAAE,IAAI,CAAC,EACzC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CACnC,CAAC;IACF,OAAO,IAAI,GAAG,MAAM,OAAO,OAAO,GAAG,GAAG,CAAC;AAC3C,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xml-format.test.d.ts","sourceRoot":"","sources":["../src/xml-format.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { wrapXml } from './xml-format.js';
|
|
3
|
+
describe('wrapXml', () => {
|
|
4
|
+
it('wraps content in XML tags', () => {
|
|
5
|
+
expect(wrapXml('knowledge', 'Hello world')).toBe('<knowledge>\nHello world\n</knowledge>');
|
|
6
|
+
});
|
|
7
|
+
it('escapes exact lowercase closing tags in content', () => {
|
|
8
|
+
const result = wrapXml('knowledge', 'payload </knowledge> injection');
|
|
9
|
+
expect(result).toBe('<knowledge>\npayload <\\/knowledge> injection\n</knowledge>');
|
|
10
|
+
});
|
|
11
|
+
it('escapes mixed-case closing tags (case-insensitive)', () => {
|
|
12
|
+
const result = wrapXml('knowledge', 'try </KNOWLEDGE> or </Knowledge>');
|
|
13
|
+
expect(result).toBe('<knowledge>\ntry <\\/KNOWLEDGE> or <\\/Knowledge>\n</knowledge>');
|
|
14
|
+
});
|
|
15
|
+
it('escapes multiple instances of the closing tag', () => {
|
|
16
|
+
const result = wrapXml('knowledge', '</knowledge> and </knowledge>');
|
|
17
|
+
expect(result).toBe('<knowledge>\n<\\/knowledge> and <\\/knowledge>\n</knowledge>');
|
|
18
|
+
});
|
|
19
|
+
it('wraps empty content', () => {
|
|
20
|
+
expect(wrapXml('git_diff', '')).toBe('<git_diff>\n\n</git_diff>');
|
|
21
|
+
});
|
|
22
|
+
it('works with different tag names', () => {
|
|
23
|
+
const result = wrapXml('lesson_added', 'Saved. </lesson_added> test');
|
|
24
|
+
expect(result).toBe('<lesson_added>\nSaved. <\\/lesson_added> test\n</lesson_added>');
|
|
25
|
+
});
|
|
26
|
+
it('escapes closing tags with internal whitespace', () => {
|
|
27
|
+
const result = wrapXml('knowledge', 'try </ knowledge> or </knowledge >');
|
|
28
|
+
expect(result).toBe('<knowledge>\ntry <\\/ knowledge> or <\\/knowledge >\n</knowledge>');
|
|
29
|
+
});
|
|
30
|
+
it('does not escape non-matching closing tags', () => {
|
|
31
|
+
const content = 'contains </other_tag> but not the target';
|
|
32
|
+
expect(wrapXml('issue_body', content)).toBe('<issue_body>\ncontains </other_tag> but not the target\n</issue_body>');
|
|
33
|
+
});
|
|
34
|
+
it('preserves multiline content', () => {
|
|
35
|
+
const content = 'line 1\nline 2\nline 3';
|
|
36
|
+
expect(wrapXml('git_status', content)).toBe('<git_status>\nline 1\nline 2\nline 3\n</git_status>');
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
//# sourceMappingURL=xml-format.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xml-format.test.js","sourceRoot":"","sources":["../src/xml-format.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE1C,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;IACvB,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;IAC7F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,gCAAgC,CAAC,CAAC;QACtE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;IACrF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,kCAAkC,CAAC,CAAC;QACxE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,+BAA+B,CAAC,CAAC;QACrE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;QAC7B,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,6BAA6B,CAAC,CAAC;QACtE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;IACxF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,oCAAoC,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,OAAO,GAAG,0CAA0C,CAAC;QAC3D,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CACzC,uEAAuE,CACxE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,OAAO,GAAG,wBAAwB,CAAC;QACzC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CACzC,qDAAqD,CACtD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|