@mmnto/totem 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunkers/markdown-chunker.test.d.ts +2 -0
- package/dist/chunkers/markdown-chunker.test.d.ts.map +1 -0
- package/dist/chunkers/markdown-chunker.test.js +114 -0
- package/dist/chunkers/markdown-chunker.test.js.map +1 -0
- package/dist/chunkers/typescript-chunker.test.d.ts +2 -0
- package/dist/chunkers/typescript-chunker.test.d.ts.map +1 -0
- package/dist/chunkers/typescript-chunker.test.js +132 -0
- package/dist/chunkers/typescript-chunker.test.js.map +1 -0
- package/dist/embedders/openai-embedder.d.ts +1 -0
- package/dist/embedders/openai-embedder.d.ts.map +1 -1
- package/dist/embedders/openai-embedder.js +24 -4
- package/dist/embedders/openai-embedder.js.map +1 -1
- package/dist/ingest/pipeline.d.ts.map +1 -1
- package/dist/ingest/pipeline.js +20 -12
- package/dist/ingest/pipeline.js.map +1 -1
- package/dist/store/lance-store.js +1 -1
- package/dist/store/lance-store.js.map +1 -1
- package/dist/store/lance-store.test.d.ts +2 -0
- package/dist/store/lance-store.test.d.ts.map +1 -0
- package/dist/store/lance-store.test.js +164 -0
- package/dist/store/lance-store.test.js.map +1 -0
- package/package.json +4 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown-chunker.test.d.ts","sourceRoot":"","sources":["../../src/chunkers/markdown-chunker.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { MarkdownChunker } from './markdown-chunker.js';
|
|
3
|
+
describe('MarkdownChunker', () => {
|
|
4
|
+
const chunker = new MarkdownChunker(() => { });
|
|
5
|
+
it('chunks by ## headings', () => {
|
|
6
|
+
const md = `# Title
|
|
7
|
+
|
|
8
|
+
Some intro text.
|
|
9
|
+
|
|
10
|
+
## Section One
|
|
11
|
+
|
|
12
|
+
Content of section one.
|
|
13
|
+
|
|
14
|
+
## Section Two
|
|
15
|
+
|
|
16
|
+
Content of section two.
|
|
17
|
+
`;
|
|
18
|
+
const chunks = chunker.chunk(md, 'docs/readme.md', 'spec');
|
|
19
|
+
expect(chunks.length).toBe(3);
|
|
20
|
+
expect(chunks[0].label).toBe('Title');
|
|
21
|
+
expect(chunks[0].content).toContain('Some intro text.');
|
|
22
|
+
expect(chunks[1].label).toBe('Section One');
|
|
23
|
+
expect(chunks[1].content).toContain('Content of section one.');
|
|
24
|
+
expect(chunks[2].label).toBe('Section Two');
|
|
25
|
+
});
|
|
26
|
+
it('chunks by ### headings', () => {
|
|
27
|
+
const md = `## Parent
|
|
28
|
+
|
|
29
|
+
Parent intro content.
|
|
30
|
+
|
|
31
|
+
### Child A
|
|
32
|
+
|
|
33
|
+
Child A content.
|
|
34
|
+
|
|
35
|
+
### Child B
|
|
36
|
+
|
|
37
|
+
Child B content.
|
|
38
|
+
`;
|
|
39
|
+
const chunks = chunker.chunk(md, 'test.md', 'spec');
|
|
40
|
+
expect(chunks.length).toBe(3);
|
|
41
|
+
expect(chunks[0].label).toBe('Parent');
|
|
42
|
+
expect(chunks[0].content).toContain('Parent intro content.');
|
|
43
|
+
expect(chunks[1].label).toBe('Child A');
|
|
44
|
+
expect(chunks[2].label).toBe('Child B');
|
|
45
|
+
});
|
|
46
|
+
it('does not split on #### headings', () => {
|
|
47
|
+
const md = `## Section
|
|
48
|
+
|
|
49
|
+
#### Subsection
|
|
50
|
+
|
|
51
|
+
Some nested content.
|
|
52
|
+
`;
|
|
53
|
+
const chunks = chunker.chunk(md, 'test.md', 'spec');
|
|
54
|
+
expect(chunks.length).toBe(1);
|
|
55
|
+
expect(chunks[0].content).toContain('Subsection');
|
|
56
|
+
expect(chunks[0].content).toContain('Some nested content.');
|
|
57
|
+
});
|
|
58
|
+
it('extracts YAML frontmatter as metadata', () => {
|
|
59
|
+
const md = `---
|
|
60
|
+
status: implemented
|
|
61
|
+
date: "2025-10-12"
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## Section
|
|
65
|
+
|
|
66
|
+
Content here.
|
|
67
|
+
`;
|
|
68
|
+
const chunks = chunker.chunk(md, 'spec.md', 'spec');
|
|
69
|
+
expect(chunks.length).toBe(1);
|
|
70
|
+
expect(chunks[0].metadata).toEqual({
|
|
71
|
+
status: 'implemented',
|
|
72
|
+
date: '2025-10-12',
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
it('sets contextPrefix with file path and section label', () => {
|
|
76
|
+
const md = `## My Section
|
|
77
|
+
|
|
78
|
+
Content.
|
|
79
|
+
`;
|
|
80
|
+
const chunks = chunker.chunk(md, 'docs/api.md', 'spec');
|
|
81
|
+
expect(chunks[0].contextPrefix).toBe('File: docs/api.md | Section: My Section');
|
|
82
|
+
});
|
|
83
|
+
it('sets correct line numbers', () => {
|
|
84
|
+
const md = `## First
|
|
85
|
+
|
|
86
|
+
Line 3 content.
|
|
87
|
+
|
|
88
|
+
## Second
|
|
89
|
+
|
|
90
|
+
Line 7 content.
|
|
91
|
+
`;
|
|
92
|
+
const chunks = chunker.chunk(md, 'test.md', 'spec');
|
|
93
|
+
expect(chunks[0].startLine).toBe(1);
|
|
94
|
+
expect(chunks[1].startLine).toBe(5);
|
|
95
|
+
});
|
|
96
|
+
it('handles empty file gracefully', () => {
|
|
97
|
+
const chunks = chunker.chunk('', 'empty.md', 'spec');
|
|
98
|
+
expect(chunks).toEqual([]);
|
|
99
|
+
});
|
|
100
|
+
it('handles file with only frontmatter', () => {
|
|
101
|
+
const md = `---
|
|
102
|
+
title: Empty
|
|
103
|
+
---
|
|
104
|
+
`;
|
|
105
|
+
const chunks = chunker.chunk(md, 'empty.md', 'spec');
|
|
106
|
+
expect(chunks).toEqual([]);
|
|
107
|
+
});
|
|
108
|
+
it('preserves chunk type from input', () => {
|
|
109
|
+
const md = `## Section\n\nContent.`;
|
|
110
|
+
const chunks = chunker.chunk(md, 'test.md', 'session_log');
|
|
111
|
+
expect(chunks[0].type).toBe('session_log');
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
//# sourceMappingURL=markdown-chunker.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"markdown-chunker.test.js","sourceRoot":"","sources":["../../src/chunkers/markdown-chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAE9C,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,EAAE,GAAG;;;;;;;;;;;CAWd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;QAE3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QAChE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;QAChC,MAAM,EAAE,GAAG;;;;;;;;;;;CAWd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,EAAE,GAAG;;;;;CAKd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,EAAE,GAAG;;;;;;;;CAQd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC;YAClC,MAAM,EAAE,aAAa;YACrB,IAAI,EAAE,YAAY;SACnB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,EAAE,GAAG;;;CAGd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;QAExD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;IACnF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,EAAE,GAAG;;;;;;;CAOd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEpD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,EAAE,GAAG;;;CAGd,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,EAAE,GAAG,wBAAwB,CAAC;QACpC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"typescript-chunker.test.d.ts","sourceRoot":"","sources":["../../src/chunkers/typescript-chunker.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { TypeScriptChunker } from './typescript-chunker.js';
|
|
3
|
+
describe('TypeScriptChunker', () => {
|
|
4
|
+
const chunker = new TypeScriptChunker();
|
|
5
|
+
it('chunks function declarations', () => {
|
|
6
|
+
const code = `export function greet(name: string): string {
|
|
7
|
+
return \`Hello, \${name}\`;
|
|
8
|
+
}
|
|
9
|
+
`;
|
|
10
|
+
const chunks = chunker.chunk(code, 'src/utils.ts', 'code');
|
|
11
|
+
expect(chunks.length).toBe(1);
|
|
12
|
+
expect(chunks[0].label).toBe('function: greet');
|
|
13
|
+
expect(chunks[0].content).toContain('export function greet');
|
|
14
|
+
});
|
|
15
|
+
it('chunks class declarations', () => {
|
|
16
|
+
const code = `export class UserService {
|
|
17
|
+
private name: string;
|
|
18
|
+
|
|
19
|
+
constructor(name: string) {
|
|
20
|
+
this.name = name;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
getName(): string {
|
|
24
|
+
return this.name;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
`;
|
|
28
|
+
const chunks = chunker.chunk(code, 'src/service.ts', 'code');
|
|
29
|
+
expect(chunks.length).toBe(1);
|
|
30
|
+
expect(chunks[0].label).toBe('class: UserService');
|
|
31
|
+
expect(chunks[0].content).toContain('getName()');
|
|
32
|
+
});
|
|
33
|
+
it('chunks interfaces', () => {
|
|
34
|
+
const code = `export interface Config {
|
|
35
|
+
port: number;
|
|
36
|
+
host: string;
|
|
37
|
+
}
|
|
38
|
+
`;
|
|
39
|
+
const chunks = chunker.chunk(code, 'src/types.ts', 'code');
|
|
40
|
+
expect(chunks.length).toBe(1);
|
|
41
|
+
expect(chunks[0].label).toBe('interface: Config');
|
|
42
|
+
});
|
|
43
|
+
it('chunks type aliases', () => {
|
|
44
|
+
const code = `export type Status = 'active' | 'inactive' | 'pending';
|
|
45
|
+
`;
|
|
46
|
+
const chunks = chunker.chunk(code, 'src/types.ts', 'code');
|
|
47
|
+
expect(chunks.length).toBe(1);
|
|
48
|
+
expect(chunks[0].label).toBe('type alias: Status');
|
|
49
|
+
});
|
|
50
|
+
it('chunks enum declarations', () => {
|
|
51
|
+
const code = `export enum Direction {
|
|
52
|
+
Up = 'UP',
|
|
53
|
+
Down = 'DOWN',
|
|
54
|
+
}
|
|
55
|
+
`;
|
|
56
|
+
const chunks = chunker.chunk(code, 'src/enums.ts', 'code');
|
|
57
|
+
expect(chunks.length).toBe(1);
|
|
58
|
+
expect(chunks[0].label).toBe('enum: Direction');
|
|
59
|
+
});
|
|
60
|
+
it('detects React components (PascalCase arrow functions)', () => {
|
|
61
|
+
const code = `export const Dashboard = () => {
|
|
62
|
+
return <div>Dashboard</div>;
|
|
63
|
+
};
|
|
64
|
+
`;
|
|
65
|
+
const chunks = chunker.chunk(code, 'src/Dashboard.tsx', 'code');
|
|
66
|
+
expect(chunks.length).toBe(1);
|
|
67
|
+
expect(chunks[0].label).toBe('component: Dashboard');
|
|
68
|
+
});
|
|
69
|
+
it('detects React hooks (use* prefix)', () => {
|
|
70
|
+
const code = `export const useAuth = () => {
|
|
71
|
+
return { user: null };
|
|
72
|
+
};
|
|
73
|
+
`;
|
|
74
|
+
const chunks = chunker.chunk(code, 'src/hooks.ts', 'code');
|
|
75
|
+
expect(chunks.length).toBe(1);
|
|
76
|
+
expect(chunks[0].label).toBe('hook: useAuth');
|
|
77
|
+
});
|
|
78
|
+
it('sets contextPrefix correctly', () => {
|
|
79
|
+
const code = `export function myFunc() {}
|
|
80
|
+
`;
|
|
81
|
+
const chunks = chunker.chunk(code, 'src/utils.ts', 'code');
|
|
82
|
+
expect(chunks[0].contextPrefix).toBe("File: src/utils.ts | Context: The 'myFunc' function");
|
|
83
|
+
});
|
|
84
|
+
it('handles multiple declarations in one file', () => {
|
|
85
|
+
const code = `export interface Options {
|
|
86
|
+
verbose: boolean;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function run(opts: Options): void {
|
|
90
|
+
console.log(opts);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export class Runner {
|
|
94
|
+
start() {}
|
|
95
|
+
}
|
|
96
|
+
`;
|
|
97
|
+
const chunks = chunker.chunk(code, 'src/runner.ts', 'code');
|
|
98
|
+
expect(chunks.length).toBe(3);
|
|
99
|
+
const labels = chunks.map((c) => c.label);
|
|
100
|
+
expect(labels).toContain('interface: Options');
|
|
101
|
+
expect(labels).toContain('function: run');
|
|
102
|
+
expect(labels).toContain('class: Runner');
|
|
103
|
+
});
|
|
104
|
+
it('handles empty file gracefully', () => {
|
|
105
|
+
const chunks = chunker.chunk('', 'empty.ts', 'code');
|
|
106
|
+
expect(chunks).toEqual([]);
|
|
107
|
+
});
|
|
108
|
+
it('skips files with only imports', () => {
|
|
109
|
+
const code = `import { foo } from './foo';
|
|
110
|
+
import { bar } from './bar';
|
|
111
|
+
`;
|
|
112
|
+
const chunks = chunker.chunk(code, 'src/index.ts', 'code');
|
|
113
|
+
expect(chunks).toEqual([]);
|
|
114
|
+
});
|
|
115
|
+
it('sets correct line numbers', () => {
|
|
116
|
+
const code = `// line 1
|
|
117
|
+
|
|
118
|
+
export function first() {
|
|
119
|
+
return 1;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export function second() {
|
|
123
|
+
return 2;
|
|
124
|
+
}
|
|
125
|
+
`;
|
|
126
|
+
const chunks = chunker.chunk(code, 'src/fns.ts', 'code');
|
|
127
|
+
expect(chunks.length).toBe(2);
|
|
128
|
+
expect(chunks[0].startLine).toBe(3);
|
|
129
|
+
expect(chunks[1].startLine).toBe(7);
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
//# sourceMappingURL=typescript-chunker.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"typescript-chunker.test.js","sourceRoot":"","sources":["../../src/chunkers/typescript-chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAE5D,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,MAAM,OAAO,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAExC,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,IAAI,GAAG;;;CAGhB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QAE3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,IAAI,GAAG;;;;;;;;;;;CAWhB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;QAE7D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACpD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mBAAmB,EAAE,GAAG,EAAE;QAC3B,MAAM,IAAI,GAAG;;;;CAIhB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QAE3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;QAC7B,MAAM,IAAI,GAAG;CAChB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QAE3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QAClC,MAAM,IAAI,GAAG;;;;CAIhB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QAE3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,IAAI,GAAG;;;CAGhB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,mBAAmB,EAAE,MAAM,CAAC,CAAC;QAEhE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,IAAI,GAAG;;;CAGhB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QAE3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,IAAI,GAAG;CAChB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QAE3D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;IAC/F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,IAAI,GAAG;;;;;;;;;;;CAWhB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;QAE5D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,IAAI,GAAG;;CAEhB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,IAAI,GAAG;;;;;;;;;CAShB,CAAC;QACE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QAEzD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai-embedder.d.ts","sourceRoot":"","sources":["../../src/embedders/openai-embedder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"openai-embedder.d.ts","sourceRoot":"","sources":["../../src/embedders/openai-embedder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAO9C,qBAAa,cAAe,YAAW,QAAQ;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAEV,KAAK,GAAE,MAAiC,EAAE,UAAU,CAAC,EAAE,MAAM;IAenE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAkBnC,cAAc;CAwB7B"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import OpenAI from 'openai';
|
|
2
2
|
const MAX_BATCH_SIZE = 2048;
|
|
3
3
|
const DEFAULT_DIMENSIONS = 1536;
|
|
4
|
+
const MAX_RETRIES = 3;
|
|
5
|
+
const INITIAL_BACKOFF_MS = 1000;
|
|
4
6
|
export class OpenAIEmbedder {
|
|
5
7
|
dimensions;
|
|
6
8
|
client;
|
|
@@ -21,10 +23,7 @@ export class OpenAIEmbedder {
|
|
|
21
23
|
const results = [];
|
|
22
24
|
for (let i = 0; i < texts.length; i += MAX_BATCH_SIZE) {
|
|
23
25
|
const batch = texts.slice(i, i + MAX_BATCH_SIZE);
|
|
24
|
-
const response = await this.
|
|
25
|
-
model: this.model,
|
|
26
|
-
input: batch,
|
|
27
|
-
});
|
|
26
|
+
const response = await this.embedWithRetry(batch);
|
|
28
27
|
const sorted = response.data.sort((a, b) => a.index - b.index);
|
|
29
28
|
for (const item of sorted) {
|
|
30
29
|
results.push(item.embedding);
|
|
@@ -32,5 +31,26 @@ export class OpenAIEmbedder {
|
|
|
32
31
|
}
|
|
33
32
|
return results;
|
|
34
33
|
}
|
|
34
|
+
async embedWithRetry(batch) {
|
|
35
|
+
let lastErr;
|
|
36
|
+
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
37
|
+
try {
|
|
38
|
+
return await this.client.embeddings.create({
|
|
39
|
+
model: this.model,
|
|
40
|
+
input: batch,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
catch (err) {
|
|
44
|
+
lastErr = err;
|
|
45
|
+
const isRetryable = err instanceof OpenAI.APIError && (err.status === 429 || err.status === 503);
|
|
46
|
+
if (!isRetryable || attempt === MAX_RETRIES)
|
|
47
|
+
break;
|
|
48
|
+
const delay = INITIAL_BACKOFF_MS * 2 ** attempt + Math.random() * 1000;
|
|
49
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const message = lastErr instanceof Error ? lastErr.message : String(lastErr);
|
|
53
|
+
throw new Error(`[Totem Error] OpenAI embedding failed after ${MAX_RETRIES + 1} attempts: ${message}`);
|
|
54
|
+
}
|
|
35
55
|
}
|
|
36
56
|
//# sourceMappingURL=openai-embedder.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai-embedder.js","sourceRoot":"","sources":["../../src/embedders/openai-embedder.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC,MAAM,OAAO,cAAc;IAChB,UAAU,CAAS;IACpB,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,QAAgB,wBAAwB,EAAE,UAAmB;QACvE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,kBAAkB,CAAC;QAEnD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,mDAAmD;gBACjD,2EAA2E,CAC9E,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"openai-embedder.js","sourceRoot":"","sources":["../../src/embedders/openai-embedder.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC,MAAM,OAAO,cAAc;IAChB,UAAU,CAAS;IACpB,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,QAAgB,wBAAwB,EAAE,UAAmB;QACvE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,kBAAkB,CAAC;QAEnD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,mDAAmD;gBACjD,2EAA2E,CAC9E,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YAElD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAC/D,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,KAAe;QAEf,IAAI,OAAgB,CAAC;QACrB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YACxD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;oBACzC,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,KAAK,EAAE,KAAK;iBACb,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,GAAG,GAAG,CAAC;gBACd,MAAM,WAAW,GACf,GAAG,YAAY,MAAM,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC;gBAC/E,IAAI,CAAC,WAAW,IAAI,OAAO,KAAK,WAAW;oBAAE,MAAM;gBACnD,MAAM,KAAK,GAAG,kBAAkB,GAAG,CAAC,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC;gBACvE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7E,MAAM,IAAI,KAAK,CACb,+CAA+C,WAAW,GAAG,CAAC,cAAc,OAAO,EAAE,CACtF,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvD,OAAO,KAAK,EAAS,WAAW,EAAE,MAAM,aAAa,CAAC;AAMtD,wBAAsB,OAAO,CAC3B,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvD,OAAO,KAAK,EAAS,WAAW,EAAE,MAAM,aAAa,CAAC;AAMtD,wBAAsB,OAAO,CAC3B,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,CAAC,CAmG9D"}
|
package/dist/ingest/pipeline.js
CHANGED
|
@@ -44,8 +44,17 @@ export async function runSync(config, options) {
|
|
|
44
44
|
filesToProcess = allFiles;
|
|
45
45
|
log(`Full sync: ${filesToProcess.length} files to process`);
|
|
46
46
|
}
|
|
47
|
-
// 4. Chunk
|
|
48
|
-
|
|
47
|
+
// 4. Chunk files and stream to LanceDB in batches (bounded memory)
|
|
48
|
+
let totalChunks = 0;
|
|
49
|
+
let buffer = [];
|
|
50
|
+
async function flushBuffer() {
|
|
51
|
+
if (buffer.length === 0)
|
|
52
|
+
return;
|
|
53
|
+
await store.insert(buffer);
|
|
54
|
+
totalChunks += buffer.length;
|
|
55
|
+
log(` Embedded ${totalChunks} chunks so far`);
|
|
56
|
+
buffer = [];
|
|
57
|
+
}
|
|
49
58
|
for (const file of filesToProcess) {
|
|
50
59
|
log(`Chunking ${file.relativePath}...`);
|
|
51
60
|
let content;
|
|
@@ -66,19 +75,18 @@ export async function runSync(config, options) {
|
|
|
66
75
|
if (incremental) {
|
|
67
76
|
await store.deleteByFile(file.relativePath);
|
|
68
77
|
}
|
|
69
|
-
|
|
78
|
+
buffer.push(...chunks);
|
|
70
79
|
log(` ${chunks.length} chunks from ${file.relativePath}`);
|
|
80
|
+
// Flush when buffer reaches batch size to keep memory bounded
|
|
81
|
+
if (buffer.length >= EMBED_BATCH_SIZE) {
|
|
82
|
+
await flushBuffer();
|
|
83
|
+
}
|
|
71
84
|
}
|
|
72
|
-
//
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const batch = allChunks.slice(i, i + EMBED_BATCH_SIZE);
|
|
76
|
-
await store.insert(batch);
|
|
77
|
-
log(` Embedded ${Math.min(i + EMBED_BATCH_SIZE, allChunks.length)}/${allChunks.length}`);
|
|
78
|
-
}
|
|
79
|
-
log(`Sync complete: ${allChunks.length} chunks from ${filesToProcess.length} files`);
|
|
85
|
+
// Flush remaining chunks
|
|
86
|
+
await flushBuffer();
|
|
87
|
+
log(`Sync complete: ${totalChunks} chunks from ${filesToProcess.length} files`);
|
|
80
88
|
return {
|
|
81
|
-
chunksProcessed:
|
|
89
|
+
chunksProcessed: totalChunks,
|
|
82
90
|
filesProcessed: filesToProcess.length,
|
|
83
91
|
};
|
|
84
92
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEnE,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,MAAmB,EACnB,OAAoB;IAEpB,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAC5C,IAAI,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IACtC,MAAM,GAAG,GAAG,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAErC,qBAAqB;IACrB,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAElD,sBAAsB;IACtB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAClD,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IAEtB,+EAA+E;IAC/E,IAAI,WAAW,IAAI,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QAC3C,GAAG,CAAC,+CAA+C,CAAC,CAAC;QACrD,WAAW,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACvF,IAAI,cAA8B,CAAC;IAEnC,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,eAAe,CAAC,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;QACzF,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,GAAG,CAAC,+CAA+C,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,cAAc,GAAG,QAAQ,CAAC;YAC1B,GAAG,CAAC,yBAAyB,cAAc,CAAC,MAAM,mBAAmB,CAAC,CAAC;QACzE,CAAC;aAAM,CAAC;YACN,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;YACzC,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;YACxE,GAAG,CAAC,qBAAqB,cAAc,CAAC,MAAM,sBAAsB,QAAQ,CAAC,MAAM,SAAS,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,+BAA+B,CAAC,CAAC;QACrC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,cAAc,GAAG,QAAQ,CAAC;QAC1B,GAAG,CAAC,cAAc,cAAc,CAAC,MAAM,mBAAmB,CAAC,CAAC;IAC9D,CAAC;IAED,
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEnE,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,MAAmB,EACnB,OAAoB;IAEpB,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAC5C,IAAI,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IACtC,MAAM,GAAG,GAAG,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAErC,qBAAqB;IACrB,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAElD,sBAAsB;IACtB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAClD,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IAEtB,+EAA+E;IAC/E,IAAI,WAAW,IAAI,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QAC3C,GAAG,CAAC,+CAA+C,CAAC,CAAC;QACrD,WAAW,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACvF,IAAI,cAA8B,CAAC;IAEnC,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,eAAe,CAAC,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;QACzF,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,GAAG,CAAC,+CAA+C,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,cAAc,GAAG,QAAQ,CAAC;YAC1B,GAAG,CAAC,yBAAyB,cAAc,CAAC,MAAM,mBAAmB,CAAC,CAAC;QACzE,CAAC;aAAM,CAAC;YACN,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;YACzC,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;YACxE,GAAG,CAAC,qBAAqB,cAAc,CAAC,MAAM,sBAAsB,QAAQ,CAAC,MAAM,SAAS,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,+BAA+B,CAAC,CAAC;QACrC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,cAAc,GAAG,QAAQ,CAAC;QAC1B,GAAG,CAAC,cAAc,cAAc,CAAC,MAAM,mBAAmB,CAAC,CAAC;IAC9D,CAAC;IAED,mEAAmE;IACnE,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,MAAM,GAAY,EAAE,CAAC;IAEzB,KAAK,UAAU,WAAW;QACxB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAChC,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3B,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;QAC7B,GAAG,CAAC,cAAc,WAAW,gBAAgB,CAAC,CAAC;QAC/C,MAAM,GAAG,EAAE,CAAC;IACd,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,GAAG,CAAC,YAAY,IAAI,CAAC,YAAY,KAAK,CAAC,CAAC;QAExC,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CACD,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CACrG,CAAC;YACF,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAE3E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,GAAG,CAAC,8BAA8B,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;YACvD,SAAS;QACX,CAAC;QAED,6EAA6E;QAC7E,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QACvB,GAAG,CAAC,KAAK,MAAM,CAAC,MAAM,gBAAgB,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAE3D,8DAA8D;QAC9D,IAAI,MAAM,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACtC,MAAM,WAAW,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,WAAW,EAAE,CAAC;IAEpB,GAAG,CAAC,kBAAkB,WAAW,gBAAgB,cAAc,CAAC,MAAM,QAAQ,CAAC,CAAC;IAEhF,OAAO;QACL,eAAe,EAAE,WAAW;QAC5B,cAAc,EAAE,cAAc,CAAC,MAAM;KACtC,CAAC;AACJ,CAAC"}
|
|
@@ -71,7 +71,7 @@ export class LanceStore {
|
|
|
71
71
|
async deleteByFile(filePath) {
|
|
72
72
|
if (!this.table)
|
|
73
73
|
return;
|
|
74
|
-
await this.table.delete(
|
|
74
|
+
await this.table.delete(`\`filePath\` = '${filePath.replace(/'/g, "''")}'`);
|
|
75
75
|
}
|
|
76
76
|
/** Drop the entire table. Used for full re-index. */
|
|
77
77
|
async reset() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lance-store.js","sourceRoot":"","sources":["../../src/store/lance-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAK5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAErD,MAAM,OAAO,UAAU;IACb,EAAE,GAA8B,IAAI,CAAC;IACrC,KAAK,GAAyB,IAAI,CAAC;IACnC,MAAM,CAAS;IACf,QAAQ,CAAW;IAE3B,YAAY,MAAc,EAAE,QAAkB;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,sEAAsE;IACtE,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE7C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QAC9C,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,KAAK,CAAC,MAAM,CAAC,MAAe;QAC1B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,mDAAmD;QACnD,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAE1E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAExD,MAAM,IAAI,GAAkB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACpD,EAAE,EAAE,UAAU,EAAE;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC;YACxC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE;SACpB,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,GAAG,IAA4C,CAAC;QAE1D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAG,CAAC,WAAW,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAE3B,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;QAE3C,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAEjE,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAY,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAEpE,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEtC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAA4B,EAAE,EAAE,CAAC,CAAC;YACpD,OAAO,EAAE,GAAG,CAAC,SAAS,CAAW;YACjC,aAAa,EAAE,GAAG,CAAC,eAAe,CAAW;YAC7C,QAAQ,EAAE,GAAG,CAAC,UAAU,CAAW;YACnC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAgB;YAChC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAW;YAC7B,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAI,GAAG,CAAC,WAAW,CAAY,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5E,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAE,GAAG,CAAC,UAAU,CAAY,IAAI,IAAI,CAA2B;SACpF,CAAC,CAAC,CAAC;IACN,CAAC;IAED,yEAAyE;IACzE,KAAK,CAAC,YAAY,CAAC,QAAgB;QACjC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"lance-store.js","sourceRoot":"","sources":["../../src/store/lance-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAK5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAErD,MAAM,OAAO,UAAU;IACb,EAAE,GAA8B,IAAI,CAAC;IACrC,KAAK,GAAyB,IAAI,CAAC;IACnC,MAAM,CAAS;IACf,QAAQ,CAAW;IAE3B,YAAY,MAAc,EAAE,QAAkB;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,sEAAsE;IACtE,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE7C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QAC9C,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,KAAK,CAAC,MAAM,CAAC,MAAe;QAC1B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,mDAAmD;QACnD,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAE1E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAExD,MAAM,IAAI,GAAkB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACpD,EAAE,EAAE,UAAU,EAAE;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC;YACxC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE;SACpB,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,GAAG,IAA4C,CAAC;QAE1D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAG,CAAC,WAAW,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAE3B,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;QAE3C,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAEjE,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAY,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAEpE,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEtC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAA4B,EAAE,EAAE,CAAC,CAAC;YACpD,OAAO,EAAE,GAAG,CAAC,SAAS,CAAW;YACjC,aAAa,EAAE,GAAG,CAAC,eAAe,CAAW;YAC7C,QAAQ,EAAE,GAAG,CAAC,UAAU,CAAW;YACnC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAgB;YAChC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAW;YAC7B,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAI,GAAG,CAAC,WAAW,CAAY,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5E,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAE,GAAG,CAAC,UAAU,CAAY,IAAI,IAAI,CAA2B;SACpF,CAAC,CAAC,CAAC;IACN,CAAC;IAED,yEAAyE;IACzE,KAAK,CAAC,YAAY,CAAC,QAAgB;QACjC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,mBAAmB,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9E,CAAC;IAED,qDAAqD;IACrD,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QAErB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QAC9C,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,kFAAkF;IAClF,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACvB,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAC7B,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,4CAA4C;IAC5C,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAEvD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAEjD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QACrE,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAW,CAAC;YAChC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;IACjC,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lance-store.test.d.ts","sourceRoot":"","sources":["../../src/store/lance-store.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as os from 'node:os';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
5
|
+
import { LanceStore } from './lance-store.js';
|
|
6
|
+
/** Deterministic fake embedder — hashes text into a fixed-dimension vector. */
|
|
7
|
+
class FakeEmbedder {
|
|
8
|
+
dimensions = 8;
|
|
9
|
+
async embed(texts) {
|
|
10
|
+
return texts.map((t) => {
|
|
11
|
+
const vec = new Array(this.dimensions).fill(0);
|
|
12
|
+
for (let i = 0; i < t.length; i++) {
|
|
13
|
+
vec[i % this.dimensions] += t.charCodeAt(i) / 1000;
|
|
14
|
+
}
|
|
15
|
+
return vec;
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
function makeChunk(overrides = {}) {
|
|
20
|
+
return {
|
|
21
|
+
content: 'test content',
|
|
22
|
+
contextPrefix: 'File: test.ts',
|
|
23
|
+
filePath: 'src/test.ts',
|
|
24
|
+
type: 'code',
|
|
25
|
+
strategy: 'typescript-ast',
|
|
26
|
+
label: 'function: test',
|
|
27
|
+
startLine: 1,
|
|
28
|
+
endLine: 10,
|
|
29
|
+
metadata: {},
|
|
30
|
+
...overrides,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
describe('LanceStore', () => {
|
|
34
|
+
let tmpDir;
|
|
35
|
+
let store;
|
|
36
|
+
let embedder;
|
|
37
|
+
beforeEach(async () => {
|
|
38
|
+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lance-test-'));
|
|
39
|
+
embedder = new FakeEmbedder();
|
|
40
|
+
store = new LanceStore(tmpDir, embedder);
|
|
41
|
+
await store.connect();
|
|
42
|
+
});
|
|
43
|
+
afterEach(async () => {
|
|
44
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
45
|
+
});
|
|
46
|
+
describe('insert + isEmpty', () => {
|
|
47
|
+
it('starts empty', async () => {
|
|
48
|
+
expect(await store.isEmpty()).toBe(true);
|
|
49
|
+
});
|
|
50
|
+
it('is not empty after insert', async () => {
|
|
51
|
+
await store.insert([makeChunk()]);
|
|
52
|
+
expect(await store.isEmpty()).toBe(false);
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
describe('stats', () => {
|
|
56
|
+
it('returns zero stats when empty', async () => {
|
|
57
|
+
const stats = await store.stats();
|
|
58
|
+
expect(stats.totalChunks).toBe(0);
|
|
59
|
+
expect(stats.byType).toEqual({});
|
|
60
|
+
});
|
|
61
|
+
it('counts chunks by type', async () => {
|
|
62
|
+
await store.insert([
|
|
63
|
+
makeChunk({ type: 'code', content: 'a' }),
|
|
64
|
+
makeChunk({ type: 'code', content: 'b' }),
|
|
65
|
+
makeChunk({ type: 'spec', content: 'c' }),
|
|
66
|
+
]);
|
|
67
|
+
const stats = await store.stats();
|
|
68
|
+
expect(stats.totalChunks).toBe(3);
|
|
69
|
+
expect(stats.byType).toEqual({ code: 2, spec: 1 });
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
describe('deleteByFile', () => {
|
|
73
|
+
it('deletes chunks for a specific file', async () => {
|
|
74
|
+
await store.insert([
|
|
75
|
+
makeChunk({ filePath: 'src/a.ts', content: 'file a' }),
|
|
76
|
+
makeChunk({ filePath: 'src/b.ts', content: 'file b' }),
|
|
77
|
+
]);
|
|
78
|
+
expect((await store.stats()).totalChunks).toBe(2);
|
|
79
|
+
await store.deleteByFile('src/a.ts');
|
|
80
|
+
const stats = await store.stats();
|
|
81
|
+
expect(stats.totalChunks).toBe(1);
|
|
82
|
+
});
|
|
83
|
+
it('does not delete chunks from other files', async () => {
|
|
84
|
+
await store.insert([
|
|
85
|
+
makeChunk({ filePath: 'src/a.ts', content: 'file a chunk 1' }),
|
|
86
|
+
makeChunk({ filePath: 'src/a.ts', content: 'file a chunk 2' }),
|
|
87
|
+
makeChunk({ filePath: 'src/b.ts', content: 'file b' }),
|
|
88
|
+
]);
|
|
89
|
+
await store.deleteByFile('src/a.ts');
|
|
90
|
+
const stats = await store.stats();
|
|
91
|
+
expect(stats.totalChunks).toBe(1);
|
|
92
|
+
});
|
|
93
|
+
it('handles paths with spaces', async () => {
|
|
94
|
+
await store.insert([
|
|
95
|
+
makeChunk({ filePath: 'src/My Folder/file.ts', content: 'spaced path' }),
|
|
96
|
+
makeChunk({ filePath: 'src/other.ts', content: 'other' }),
|
|
97
|
+
]);
|
|
98
|
+
await store.deleteByFile('src/My Folder/file.ts');
|
|
99
|
+
const stats = await store.stats();
|
|
100
|
+
expect(stats.totalChunks).toBe(1);
|
|
101
|
+
});
|
|
102
|
+
it('handles paths with single quotes', async () => {
|
|
103
|
+
await store.insert([
|
|
104
|
+
makeChunk({ filePath: "src/user's-config.ts", content: 'quoted path' }),
|
|
105
|
+
makeChunk({ filePath: 'src/other.ts', content: 'other' }),
|
|
106
|
+
]);
|
|
107
|
+
await store.deleteByFile("src/user's-config.ts");
|
|
108
|
+
const stats = await store.stats();
|
|
109
|
+
expect(stats.totalChunks).toBe(1);
|
|
110
|
+
});
|
|
111
|
+
it('handles camelCase paths', async () => {
|
|
112
|
+
await store.insert([
|
|
113
|
+
makeChunk({ filePath: 'src/myComponent/CamelCase.tsx', content: 'camel' }),
|
|
114
|
+
makeChunk({ filePath: 'src/other.ts', content: 'other' }),
|
|
115
|
+
]);
|
|
116
|
+
await store.deleteByFile('src/myComponent/CamelCase.tsx');
|
|
117
|
+
const stats = await store.stats();
|
|
118
|
+
expect(stats.totalChunks).toBe(1);
|
|
119
|
+
});
|
|
120
|
+
it('is a no-op when table is empty', async () => {
|
|
121
|
+
// Should not throw
|
|
122
|
+
await store.deleteByFile('nonexistent.ts');
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
describe('search', () => {
|
|
126
|
+
it('returns empty when table is empty', async () => {
|
|
127
|
+
const results = await store.search({ query: 'test' });
|
|
128
|
+
expect(results).toEqual([]);
|
|
129
|
+
});
|
|
130
|
+
it('returns results matching query', async () => {
|
|
131
|
+
await store.insert([
|
|
132
|
+
makeChunk({ content: 'handles user authentication', label: 'auth' }),
|
|
133
|
+
makeChunk({ content: 'renders the dashboard component', label: 'dashboard' }),
|
|
134
|
+
]);
|
|
135
|
+
const results = await store.search({ query: 'authentication login', maxResults: 2 });
|
|
136
|
+
expect(results.length).toBeGreaterThan(0);
|
|
137
|
+
expect(results[0].score).toBeGreaterThan(0);
|
|
138
|
+
});
|
|
139
|
+
it('filters by type', async () => {
|
|
140
|
+
await store.insert([
|
|
141
|
+
makeChunk({ type: 'code', content: 'code content alpha' }),
|
|
142
|
+
makeChunk({ type: 'spec', content: 'spec content beta' }),
|
|
143
|
+
]);
|
|
144
|
+
const results = await store.search({ query: 'content', typeFilter: 'spec', maxResults: 10 });
|
|
145
|
+
expect(results.every((r) => r.type === 'spec')).toBe(true);
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
describe('reset', () => {
|
|
149
|
+
it('drops all data', async () => {
|
|
150
|
+
await store.insert([makeChunk()]);
|
|
151
|
+
expect(await store.isEmpty()).toBe(false);
|
|
152
|
+
await store.reset();
|
|
153
|
+
expect(await store.isEmpty()).toBe(true);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
describe('reconnect', () => {
|
|
157
|
+
it('re-opens the connection', async () => {
|
|
158
|
+
await store.insert([makeChunk()]);
|
|
159
|
+
await store.reconnect();
|
|
160
|
+
expect(await store.isEmpty()).toBe(false);
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
});
|
|
164
|
+
//# sourceMappingURL=lance-store.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lance-store.test.js","sourceRoot":"","sources":["../../src/store/lance-store.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAIrE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,+EAA+E;AAC/E,MAAM,YAAY;IACP,UAAU,GAAG,CAAC,CAAC;IAExB,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACrB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAa,CAAC;YAC3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YACrD,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,SAAS,SAAS,CAAC,YAA4B,EAAE;IAC/C,OAAO;QACL,OAAO,EAAE,cAAc;QACvB,aAAa,EAAE,eAAe;QAC9B,QAAQ,EAAE,aAAa;QACvB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,gBAAgB;QAC1B,KAAK,EAAE,gBAAgB;QACvB,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,EAAE;QACX,QAAQ,EAAE,EAAE;QACZ,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,IAAI,MAAc,CAAC;IACnB,IAAI,KAAiB,CAAC;IACtB,IAAI,QAAsB,CAAC;IAE3B,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;QAC/D,QAAQ,GAAG,IAAI,YAAY,EAAE,CAAC;QAC9B,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YAC5B,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YACzC,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;YACrC,MAAM,KAAK,CAAC,MAAM,CAAC;gBACjB,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;gBACzC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;gBACzC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;aAC1C,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,KAAK,CAAC,MAAM,CAAC;gBACjB,SAAS,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;gBACtD,SAAS,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;aACvD,CAAC,CAAC;YAEH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAElD,MAAM,KAAK,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YAErC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,KAAK,CAAC,MAAM,CAAC;gBACjB,SAAS,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;gBAC9D,SAAS,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;gBAC9D,SAAS,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;aACvD,CAAC,CAAC;YAEH,MAAM,KAAK,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YAErC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YACzC,MAAM,KAAK,CAAC,MAAM,CAAC;gBACjB,SAAS,CAAC,EAAE,QAAQ,EAAE,uBAAuB,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;gBACxE,SAAS,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;aAC1D,CAAC,CAAC;YAEH,MAAM,KAAK,CAAC,YAAY,CAAC,uBAAuB,CAAC,CAAC;YAElD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,KAAK,CAAC,MAAM,CAAC;gBACjB,SAAS,CAAC,EAAE,QAAQ,EAAE,sBAAsB,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;gBACvE,SAAS,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;aAC1D,CAAC,CAAC;YAEH,MAAM,KAAK,CAAC,YAAY,CAAC,sBAAsB,CAAC,CAAC;YAEjD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,KAAK,CAAC,MAAM,CAAC;gBACjB,SAAS,CAAC,EAAE,QAAQ,EAAE,+BAA+B,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;gBAC1E,SAAS,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;aAC1D,CAAC,CAAC;YAEH,MAAM,KAAK,CAAC,YAAY,CAAC,+BAA+B,CAAC,CAAC;YAE1D,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,mBAAmB;YACnB,MAAM,KAAK,CAAC,YAAY,CAAC,gBAAgB,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACtD,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,KAAK,CAAC,MAAM,CAAC;gBACjB,SAAS,CAAC,EAAE,OAAO,EAAE,6BAA6B,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;gBACpE,SAAS,CAAC,EAAE,OAAO,EAAE,iCAAiC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;aAC9E,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;YACrF,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC1C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iBAAiB,EAAE,KAAK,IAAI,EAAE;YAC/B,MAAM,KAAK,CAAC,MAAM,CAAC;gBACjB,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC;gBAC1D,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC;aAC1D,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7F,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,gBAAgB,EAAE,KAAK,IAAI,EAAE;YAC9B,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE1C,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YAClC,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mmnto/totem",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Persistent memory and context layer for AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -24,7 +24,8 @@
|
|
|
24
24
|
"zod": "^3.24.0"
|
|
25
25
|
},
|
|
26
26
|
"devDependencies": {
|
|
27
|
-
"@types/mdast": "^4.0.0"
|
|
27
|
+
"@types/mdast": "^4.0.0",
|
|
28
|
+
"vitest": "^3.0.0"
|
|
28
29
|
},
|
|
29
30
|
"files": [
|
|
30
31
|
"dist"
|
|
@@ -48,6 +49,6 @@
|
|
|
48
49
|
"dev": "tsc --watch",
|
|
49
50
|
"clean": "rm -rf dist",
|
|
50
51
|
"lint": "eslint src/",
|
|
51
|
-
"test": "
|
|
52
|
+
"test": "vitest run"
|
|
52
53
|
}
|
|
53
54
|
}
|