@milo4jo/contextkit 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +237 -72
- package/dist/commands/index-cmd.d.ts.map +1 -1
- package/dist/commands/index-cmd.js +10 -2
- package/dist/commands/index-cmd.js.map +1 -1
- package/dist/commands/mcp.d.ts +8 -0
- package/dist/commands/mcp.d.ts.map +1 -0
- package/dist/commands/mcp.js +23 -0
- package/dist/commands/mcp.js.map +1 -0
- package/dist/commands/watch.d.ts +8 -0
- package/dist/commands/watch.d.ts.map +1 -0
- package/dist/commands/watch.js +171 -0
- package/dist/commands/watch.js.map +1 -0
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +11 -2
- package/dist/db/index.js.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/indexer/chunker.d.ts.map +1 -1
- package/dist/indexer/chunker.js +14 -15
- package/dist/indexer/chunker.js.map +1 -1
- package/dist/indexer/discovery.d.ts +7 -0
- package/dist/indexer/discovery.d.ts.map +1 -1
- package/dist/indexer/discovery.js +9 -0
- package/dist/indexer/discovery.js.map +1 -1
- package/dist/indexer/index.d.ts +17 -3
- package/dist/indexer/index.d.ts.map +1 -1
- package/dist/indexer/index.js +135 -24
- package/dist/indexer/index.js.map +1 -1
- package/dist/mcp/index.d.ts +7 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +7 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/server.d.ts +20 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +252 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp-server.d.ts +18 -0
- package/dist/mcp-server.d.ts.map +1 -0
- package/dist/mcp-server.js +22 -0
- package/dist/mcp-server.js.map +1 -0
- package/dist/selector/budget.d.ts +2 -0
- package/dist/selector/budget.d.ts.map +1 -1
- package/dist/selector/budget.js +19 -6
- package/dist/selector/budget.js.map +1 -1
- package/dist/selector/formatter.d.ts.map +1 -1
- package/dist/selector/formatter.js +7 -5
- package/dist/selector/formatter.js.map +1 -1
- package/dist/selector/scoring.d.ts +15 -5
- package/dist/selector/scoring.d.ts.map +1 -1
- package/dist/selector/scoring.js +193 -19
- package/dist/selector/scoring.js.map +1 -1
- package/dist/utils/streams.d.ts +4 -0
- package/dist/utils/streams.d.ts.map +1 -1
- package/dist/utils/streams.js +7 -0
- package/dist/utils/streams.js.map +1 -1
- package/package.json +19 -7
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Watch Command
|
|
3
|
+
*
|
|
4
|
+
* Watches source directories and auto-reindexes on changes.
|
|
5
|
+
*/
|
|
6
|
+
import { Command } from 'commander';
|
|
7
|
+
import chokidar from 'chokidar';
|
|
8
|
+
import { resolve, relative } from 'path';
|
|
9
|
+
import { loadConfig, ensureInitialized } from '../config/index.js';
|
|
10
|
+
import { openDatabase } from '../db/index.js';
|
|
11
|
+
import { indexSources } from '../indexer/index.js';
|
|
12
|
+
import { writeMessage, writeSuccess, writeWarning, writeDim } from '../utils/streams.js';
|
|
13
|
+
import { formatHighlight, formatDim } from '../utils/format.js';
|
|
14
|
+
/** Default debounce time in milliseconds */
|
|
15
|
+
const DEFAULT_DEBOUNCE_MS = 1000;
|
|
16
|
+
export const watchCommand = new Command('watch')
|
|
17
|
+
.description('Watch sources and auto-reindex on changes')
|
|
18
|
+
.option('-d, --debounce <ms>', 'Debounce time in milliseconds', String(DEFAULT_DEBOUNCE_MS))
|
|
19
|
+
.action(async (options) => {
|
|
20
|
+
ensureInitialized();
|
|
21
|
+
const config = loadConfig();
|
|
22
|
+
const debounceMs = parseInt(options.debounce) || DEFAULT_DEBOUNCE_MS;
|
|
23
|
+
if (config.sources.length === 0) {
|
|
24
|
+
writeWarning('No sources configured');
|
|
25
|
+
writeMessage('Add sources first with: contextkit source add <path>');
|
|
26
|
+
process.exit(1);
|
|
27
|
+
}
|
|
28
|
+
const cwd = process.cwd();
|
|
29
|
+
// Build watch patterns from sources
|
|
30
|
+
const watchPaths = [];
|
|
31
|
+
for (const source of config.sources) {
|
|
32
|
+
const sourcePath = resolve(cwd, source.path);
|
|
33
|
+
watchPaths.push(sourcePath);
|
|
34
|
+
}
|
|
35
|
+
writeMessage('');
|
|
36
|
+
writeMessage(`${formatHighlight('👁️ Watching')} ${config.sources.length} source(s) for changes...`);
|
|
37
|
+
writeMessage(formatDim(` Debounce: ${debounceMs}ms`));
|
|
38
|
+
writeMessage('');
|
|
39
|
+
for (const source of config.sources) {
|
|
40
|
+
writeMessage(` • ${source.id}: ${source.path}`);
|
|
41
|
+
}
|
|
42
|
+
writeMessage('');
|
|
43
|
+
writeMessage(formatDim('Press Ctrl+C to stop watching.'));
|
|
44
|
+
writeMessage('');
|
|
45
|
+
// Track pending changes
|
|
46
|
+
const pendingChanges = new Set();
|
|
47
|
+
let debounceTimer = null;
|
|
48
|
+
let isIndexing = false;
|
|
49
|
+
/**
|
|
50
|
+
* Run indexing for pending changes
|
|
51
|
+
*/
|
|
52
|
+
async function runIndex() {
|
|
53
|
+
if (isIndexing)
|
|
54
|
+
return;
|
|
55
|
+
if (pendingChanges.size === 0)
|
|
56
|
+
return;
|
|
57
|
+
isIndexing = true;
|
|
58
|
+
const changedFiles = Array.from(pendingChanges);
|
|
59
|
+
pendingChanges.clear();
|
|
60
|
+
const timestamp = new Date().toLocaleTimeString();
|
|
61
|
+
writeMessage(`${formatDim(`[${timestamp}]`)} Detected ${changedFiles.length} change(s), reindexing...`);
|
|
62
|
+
const db = openDatabase();
|
|
63
|
+
try {
|
|
64
|
+
const stats = await indexSources(config.sources, cwd, db, {
|
|
65
|
+
chunkSize: config.settings.chunk_size,
|
|
66
|
+
chunkOverlap: config.settings.chunk_overlap,
|
|
67
|
+
});
|
|
68
|
+
if (stats.filesChanged > 0) {
|
|
69
|
+
writeSuccess(`Indexed ${stats.filesChanged} changed file(s) → ${stats.chunks} chunks`);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
writeMessage(formatDim('No changes detected.'));
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
writeWarning(`Index error: ${error instanceof Error ? error.message : String(error)}`);
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
db.close();
|
|
80
|
+
isIndexing = false;
|
|
81
|
+
// If more changes accumulated while indexing, run again
|
|
82
|
+
if (pendingChanges.size > 0) {
|
|
83
|
+
scheduleIndex();
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Schedule an index run with debouncing
|
|
89
|
+
*/
|
|
90
|
+
function scheduleIndex() {
|
|
91
|
+
if (debounceTimer) {
|
|
92
|
+
clearTimeout(debounceTimer);
|
|
93
|
+
}
|
|
94
|
+
debounceTimer = setTimeout(runIndex, debounceMs);
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Handle file change event
|
|
98
|
+
*/
|
|
99
|
+
function handleChange(filePath, eventType) {
|
|
100
|
+
// Get relative path for display
|
|
101
|
+
const relPath = relative(cwd, filePath);
|
|
102
|
+
// Check if file matches any source patterns
|
|
103
|
+
let matchesSource = false;
|
|
104
|
+
for (const source of config.sources) {
|
|
105
|
+
const sourcePath = resolve(cwd, source.path);
|
|
106
|
+
if (filePath.startsWith(sourcePath)) {
|
|
107
|
+
// Check include patterns
|
|
108
|
+
for (const pattern of source.patterns.include) {
|
|
109
|
+
// Simple check - could be more sophisticated
|
|
110
|
+
const ext = pattern.replace('**/*', '').replace('*', '');
|
|
111
|
+
if (filePath.endsWith(ext) || pattern === '**/*') {
|
|
112
|
+
matchesSource = true;
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
// Check exclude patterns
|
|
117
|
+
if (matchesSource) {
|
|
118
|
+
for (const pattern of source.patterns.exclude) {
|
|
119
|
+
if (pattern.includes('node_modules') && filePath.includes('node_modules')) {
|
|
120
|
+
matchesSource = false;
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
if (pattern.includes('.test.') && filePath.includes('.test.')) {
|
|
124
|
+
matchesSource = false;
|
|
125
|
+
break;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if (matchesSource)
|
|
131
|
+
break;
|
|
132
|
+
}
|
|
133
|
+
if (matchesSource) {
|
|
134
|
+
pendingChanges.add(filePath);
|
|
135
|
+
writeDim(` ${eventType}: ${relPath}`);
|
|
136
|
+
scheduleIndex();
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
// Setup file watcher
|
|
140
|
+
const watcher = chokidar.watch(watchPaths, {
|
|
141
|
+
ignored: [
|
|
142
|
+
'**/node_modules/**',
|
|
143
|
+
'**/.git/**',
|
|
144
|
+
'**/.contextkit/**',
|
|
145
|
+
'**/dist/**',
|
|
146
|
+
'**/build/**',
|
|
147
|
+
'**/*.test.*',
|
|
148
|
+
'**/*.spec.*',
|
|
149
|
+
],
|
|
150
|
+
persistent: true,
|
|
151
|
+
ignoreInitial: true,
|
|
152
|
+
awaitWriteFinish: {
|
|
153
|
+
stabilityThreshold: 100,
|
|
154
|
+
pollInterval: 100,
|
|
155
|
+
},
|
|
156
|
+
});
|
|
157
|
+
watcher
|
|
158
|
+
.on('add', (path) => handleChange(path, 'added'))
|
|
159
|
+
.on('change', (path) => handleChange(path, 'changed'))
|
|
160
|
+
.on('unlink', (path) => handleChange(path, 'removed'));
|
|
161
|
+
// Handle graceful shutdown
|
|
162
|
+
process.on('SIGINT', () => {
|
|
163
|
+
writeMessage('');
|
|
164
|
+
writeMessage('Stopping watch...');
|
|
165
|
+
watcher.close();
|
|
166
|
+
process.exit(0);
|
|
167
|
+
});
|
|
168
|
+
// Keep process running
|
|
169
|
+
await new Promise(() => { });
|
|
170
|
+
});
|
|
171
|
+
//# sourceMappingURL=watch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"watch.js","sourceRoot":"","sources":["../../src/commands/watch.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAEhE,4CAA4C;AAC5C,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC;KAC7C,WAAW,CAAC,2CAA2C,CAAC;KACxD,MAAM,CAAC,qBAAqB,EAAE,+BAA+B,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;KAC3F,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,iBAAiB,EAAE,CAAC;IAEpB,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,mBAAmB,CAAC;IAErE,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,YAAY,CAAC,uBAAuB,CAAC,CAAC;QACtC,YAAY,CAAC,sDAAsD,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE1B,oCAAoC;IACpC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7C,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAED,YAAY,CAAC,EAAE,CAAC,CAAC;IACjB,YAAY,CAAC,GAAG,eAAe,CAAC,eAAe,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,2BAA2B,CAAC,CAAC;IACtG,YAAY,CAAC,SAAS,CAAC,gBAAgB,UAAU,IAAI,CAAC,CAAC,CAAC;IACxD,YAAY,CAAC,EAAE,CAAC,CAAC;IAEjB,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,YAAY,CAAC,QAAQ,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,YAAY,CAAC,EAAE,CAAC,CAAC;IACjB,YAAY,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC,CAAC;IAC1D,YAAY,CAAC,EAAE,CAAC,CAAC;IAEjB,wBAAwB;IACxB,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,IAAI,aAAa,GAA0B,IAAI,CAAC;IAChD,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB;;OAEG;IACH,KAAK,UAAU,QAAQ;QACrB,IAAI,UAAU;YAAE,OAAO;QACvB,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO;QAEtC,UAAU,GAAG,IAAI,CAAC;QAClB,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChD,cAAc,CAAC,KAAK,EAAE,CAAC;QAEvB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,CAAC;QAClD,YAAY,CAAC,GAAG,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,aAAa,YAAY,CAAC,MAAM,2BAA2B,CAAC,CAAC;QAExG,MAAM,EAAE,GAAG,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,YAAY,CAC9B,MAAM,CAAC,OAAO,EACd,GAAG,EACH,EAAE,EACF;gBACE,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU;gBACrC,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa;aAC5C,CACF,CAAC;YAEF,IAAI,KAAK,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;gBAC3B,YAAY,CAAC,WAAW,KAAK,CAAC,YAAY,sBAAsB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;YACzF,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,gBAAgB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,UAAU,GAAG,KAAK,CAAC;YAEnB,wDAAwD;YACxD,IAAI,cAAc,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC5B,aAAa,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS,aAAa;QACpB,IAAI,aAAa,EAAE,CAAC;YAClB,YAAY,CAAC,aAAa,CAAC,CAAC;QAC9B,CAAC;QACD,aAAa,GAAG,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,SAAS,YAAY,CAAC,QAAgB,EAAE,SAAiB;QACvD,gCAAgC;QAChC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAExC,4CAA4C;QAC5C,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC7C,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpC,yBAAyB;gBACzB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;oBAC9C,6CAA6C;oBAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBACzD,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;wBACjD,aAAa,GAAG,IAAI,CAAC;wBACrB,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,yBAAyB;gBACzB,IAAI,aAAa,EAAE,CAAC;oBAClB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;wBAC9C,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;4BAC1E,aAAa,GAAG,KAAK,CAAC;4BACtB,MAAM;wBACR,CAAC;wBACD,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;4BAC9D,aAAa,GAAG,KAAK,CAAC;4BACtB,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,aAAa;gBAAE,MAAM;QAC3B,CAAC;QAED,IAAI,aAAa,EAAE,CAAC;YAClB,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC7B,QAAQ,CAAC,MAAM,SAAS,KAAK,OAAO,EAAE,CAAC,CAAC;YACxC,aAAa,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE;QACzC,OAAO,EAAE;YACP,oBAAoB;YACpB,YAAY;YACZ,mBAAmB;YACnB,YAAY;YACZ,aAAa;YACb,aAAa;YACb,aAAa;SACd;QACD,UAAU,EAAE,IAAI;QAChB,aAAa,EAAE,IAAI;QACnB,gBAAgB,EAAE;YAChB,kBAAkB,EAAE,GAAG;YACvB,YAAY,EAAE,GAAG;SAClB;KACF,CAAC,CAAC;IAEH,OAAO;SACJ,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;SAChD,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;SACrD,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IAEzD,2BAA2B;IAC3B,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACxB,YAAY,CAAC,EAAE,CAAC,CAAC;QACjB,YAAY,CAAC,mBAAmB,CAAC,CAAC;QAClC,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,uBAAuB;IACvB,MAAM,IAAI,OAAO,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AAC9B,CAAC,CAAC,CAAC"}
|
package/dist/db/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AA+CtC;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAU9D;AAED;;GAEG;AACH,wBAAgB,YAAY,IAAI,QAAQ,CAAC,QAAQ,CAKhD;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAEzD"}
|
package/dist/db/index.js
CHANGED
|
@@ -14,8 +14,15 @@ CREATE TABLE IF NOT EXISTS sources (
|
|
|
14
14
|
indexed_at TIMESTAMP
|
|
15
15
|
);
|
|
16
16
|
|
|
17
|
-
--
|
|
18
|
-
|
|
17
|
+
-- Files table (for incremental indexing)
|
|
18
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
19
|
+
id TEXT PRIMARY KEY,
|
|
20
|
+
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
|
21
|
+
file_path TEXT NOT NULL,
|
|
22
|
+
content_hash TEXT NOT NULL,
|
|
23
|
+
indexed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
24
|
+
UNIQUE(source_id, file_path)
|
|
25
|
+
);
|
|
19
26
|
|
|
20
27
|
-- Chunks table
|
|
21
28
|
CREATE TABLE IF NOT EXISTS chunks (
|
|
@@ -33,6 +40,8 @@ CREATE TABLE IF NOT EXISTS chunks (
|
|
|
33
40
|
-- Indexes for performance
|
|
34
41
|
CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source_id);
|
|
35
42
|
CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_path);
|
|
43
|
+
CREATE INDEX IF NOT EXISTS idx_files_source ON files(source_id);
|
|
44
|
+
CREATE INDEX IF NOT EXISTS idx_files_path ON files(source_id, file_path);
|
|
36
45
|
`;
|
|
37
46
|
/**
|
|
38
47
|
* Initialize the database with schema
|
package/dist/db/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C;;GAEG;AACH,MAAM,MAAM,GAAG
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C;;GAEG;AACH,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuCd,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEhC,sBAAsB;IACtB,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAE/B,gBAAgB;IAChB,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAChC,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,EAAqB;IACjD,EAAE,CAAC,KAAK,EAAE,CAAC;AACb,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -7,6 +7,8 @@ import { initCommand } from './commands/init.js';
|
|
|
7
7
|
import { sourceCommand } from './commands/source/index.js';
|
|
8
8
|
import { indexCommand } from './commands/index-cmd.js';
|
|
9
9
|
import { selectCommand } from './commands/select.js';
|
|
10
|
+
import { mcpCommand } from './commands/mcp.js';
|
|
11
|
+
import { watchCommand } from './commands/watch.js';
|
|
10
12
|
import { ContextKitError, InvalidUsageError } from './errors/index.js';
|
|
11
13
|
import { writeError, writeMessage } from './utils/streams.js';
|
|
12
14
|
// Get version from package.json
|
|
@@ -60,6 +62,8 @@ program.addCommand(initCommand);
|
|
|
60
62
|
program.addCommand(sourceCommand);
|
|
61
63
|
program.addCommand(indexCommand);
|
|
62
64
|
program.addCommand(selectCommand);
|
|
65
|
+
program.addCommand(mcpCommand);
|
|
66
|
+
program.addCommand(watchCommand);
|
|
63
67
|
// Default action when no command given
|
|
64
68
|
program.action(() => {
|
|
65
69
|
console.log(`
|
|
@@ -74,8 +78,10 @@ Quick Start:
|
|
|
74
78
|
Commands:
|
|
75
79
|
init Initialize ContextKit in current directory
|
|
76
80
|
source Manage source directories
|
|
77
|
-
index Index all sources (
|
|
81
|
+
index Index all sources (incremental by default)
|
|
78
82
|
select Select context for a query
|
|
83
|
+
watch Watch sources and auto-reindex on changes
|
|
84
|
+
mcp Start MCP server for AI assistants
|
|
79
85
|
|
|
80
86
|
Global Options:
|
|
81
87
|
--json Output as JSON
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE9D,gCAAgC;AAChC,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AACrF,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;AAE5B,+BAA+B;AAC/B,MAAM,UAAU,GAAG,CAAC,CAAC;AACrB,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B;;GAEG;AACH,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,KAAK,YAAY,eAAe,EAAE,CAAC;QACrC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAE1B,MAAM,QAAQ,GAAG,KAAK,YAAY,iBAAiB,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,UAAU,CAAC;QAEtF,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAED,gDAAgD;IAChD,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACtB,YAAY,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,UAAU,CAAC,8BAA8B,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AAC3B,CAAC;AAED,wBAAwB;AACxB,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAC;AAC7C,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;AAE9C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,YAAY,CAAC;KAClB,WAAW,CAAC,kCAAkC,CAAC;KAC/C,OAAO,CAAC,OAAO,EAAE,eAAe,EAAE,qBAAqB,CAAC;KACxD,kBAAkB,EAAE;KACpB,aAAa,CAAC;IACb,eAAe,EAAE,IAAI;IACrB,cAAc,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE;CACpC,CAAC,CAAC;AAEL,iBAAiB;AACjB,OAAO;KACJ,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KAClC,MAAM,CAAC,SAAS,EAAE,qCAAqC,CAAC;KACxD,MAAM,CAAC,SAAS,EAAE,+BAA+B,CAAC,CAAC;AAEtD,oBAAoB;AACpB,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;AAChC,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;AAClC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;AACjC,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE9D,gCAAgC;AAChC,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AACrF,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;AAE5B,+BAA+B;AAC/B,MAAM,UAAU,GAAG,CAAC,CAAC;AACrB,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B;;GAEG;AACH,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,KAAK,YAAY,eAAe,EAAE,CAAC;QACrC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAE1B,MAAM,QAAQ,GAAG,KAAK,YAAY,iBAAiB,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,UAAU,CAAC;QAEtF,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAED,gDAAgD;IAChD,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACtB,YAAY,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,UAAU,CAAC,8BAA8B,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AAC3B,CAAC;AAED,wBAAwB;AACxB,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAC;AAC7C,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;AAE9C,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,YAAY,CAAC;KAClB,WAAW,CAAC,kCAAkC,CAAC;KAC/C,OAAO,CAAC,OAAO,EAAE,eAAe,EAAE,qBAAqB,CAAC;KACxD,kBAAkB,EAAE;KACpB,aAAa,CAAC;IACb,eAAe,EAAE,IAAI;IACrB,cAAc,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE;CACpC,CAAC,CAAC;AAEL,iBAAiB;AACjB,OAAO;KACJ,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KAClC,MAAM,CAAC,SAAS,EAAE,qCAAqC,CAAC;KACxD,MAAM,CAAC,SAAS,EAAE,+BAA+B,CAAC,CAAC;AAEtD,oBAAoB;AACpB,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;AAChC,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;AAClC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;AACjC,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;AAClC,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;AAC/B,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;AAEjC,uCAAuC;AACvC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE;IAClB,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;;;;;;;;;;;;;CAwBb,CAAC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/indexer/chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;
|
|
1
|
+
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/indexer/chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAErD,2CAA2C;AAC3C,MAAM,WAAW,KAAK;IACpB,sBAAsB;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,mCAAmC;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,kBAAkB;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,uBAAuB;AACvB,MAAM,WAAW,YAAY;IAC3B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;CACtB;AAUD;;GAEG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEhD;AAkBD;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,cAAc,EAAE,OAAO,GAAE,YAA8B,GAAG,KAAK,EAAE,CAgEhG;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,KAAK,EAAE,cAAc,EAAE,EACvB,OAAO,GAAE,YAA8B,GACtC,KAAK,EAAE,CAST"}
|
package/dist/indexer/chunker.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Splits files into chunks for embedding.
|
|
5
5
|
* Uses line-based chunking with configurable size and overlap.
|
|
6
6
|
*/
|
|
7
|
+
import { createHash } from 'crypto';
|
|
7
8
|
import { encodingForModel } from 'js-tiktoken';
|
|
8
9
|
const DEFAULT_OPTIONS = {
|
|
9
10
|
chunkSize: 500,
|
|
@@ -18,18 +19,14 @@ export function countTokens(text) {
|
|
|
18
19
|
return encoder.encode(text).length;
|
|
19
20
|
}
|
|
20
21
|
/**
|
|
21
|
-
* Generate a unique chunk ID
|
|
22
|
+
* Generate a unique chunk ID using SHA-256 hash
|
|
23
|
+
* Includes both startLine and endLine to handle edge cases where
|
|
24
|
+
* high overlap values could result in same startLine for different chunks.
|
|
22
25
|
*/
|
|
23
|
-
function generateChunkId(sourceId, filePath, startLine) {
|
|
24
|
-
const base = `${sourceId}:${filePath}:${startLine}`;
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
for (let i = 0; i < base.length; i++) {
|
|
28
|
-
const char = base.charCodeAt(i);
|
|
29
|
-
hash = (hash << 5) - hash + char;
|
|
30
|
-
hash = hash & hash; // Convert to 32-bit integer
|
|
31
|
-
}
|
|
32
|
-
return `chunk_${Math.abs(hash).toString(36)}`;
|
|
26
|
+
function generateChunkId(sourceId, filePath, startLine, endLine) {
|
|
27
|
+
const base = `${sourceId}:${filePath}:${startLine}:${endLine}`;
|
|
28
|
+
const hash = createHash('sha256').update(base).digest('hex').slice(0, 16);
|
|
29
|
+
return `chunk_${hash}`;
|
|
33
30
|
}
|
|
34
31
|
/**
|
|
35
32
|
* Chunk a single file into pieces
|
|
@@ -46,13 +43,14 @@ export function chunkFile(file, options = DEFAULT_OPTIONS) {
|
|
|
46
43
|
// If adding this line exceeds chunk size, save current chunk
|
|
47
44
|
if (currentTokens + lineTokens > options.chunkSize && currentLines.length > 0) {
|
|
48
45
|
const content = currentLines.join('\n');
|
|
46
|
+
const endLine = startLine + currentLines.length - 1;
|
|
49
47
|
chunks.push({
|
|
50
|
-
id: generateChunkId(file.sourceId, file.relativePath, startLine),
|
|
48
|
+
id: generateChunkId(file.sourceId, file.relativePath, startLine, endLine),
|
|
51
49
|
sourceId: file.sourceId,
|
|
52
50
|
filePath: file.relativePath,
|
|
53
51
|
content,
|
|
54
52
|
startLine,
|
|
55
|
-
endLine
|
|
53
|
+
endLine,
|
|
56
54
|
tokens: currentTokens,
|
|
57
55
|
});
|
|
58
56
|
// Calculate overlap: keep last N tokens worth of lines
|
|
@@ -76,13 +74,14 @@ export function chunkFile(file, options = DEFAULT_OPTIONS) {
|
|
|
76
74
|
// Don't forget the last chunk
|
|
77
75
|
if (currentLines.length > 0) {
|
|
78
76
|
const content = currentLines.join('\n');
|
|
77
|
+
const endLine = startLine + currentLines.length - 1;
|
|
79
78
|
chunks.push({
|
|
80
|
-
id: generateChunkId(file.sourceId, file.relativePath, startLine),
|
|
79
|
+
id: generateChunkId(file.sourceId, file.relativePath, startLine, endLine),
|
|
81
80
|
sourceId: file.sourceId,
|
|
82
81
|
filePath: file.relativePath,
|
|
83
82
|
content,
|
|
84
83
|
startLine,
|
|
85
|
-
endLine
|
|
84
|
+
endLine,
|
|
86
85
|
tokens: countTokens(content),
|
|
87
86
|
});
|
|
88
87
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/indexer/chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AA6B/C,MAAM,eAAe,GAAiB;IACpC,SAAS,EAAE,GAAG;IACd,YAAY,EAAE,EAAE;CACjB,CAAC;AAEF,qDAAqD;AACrD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;AAE1C;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/indexer/chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AA6B/C,MAAM,eAAe,GAAiB;IACpC,SAAS,EAAE,GAAG;IACd,YAAY,EAAE,EAAE;CACjB,CAAC;AAEF,qDAAqD;AACrD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;AAE1C;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CACtB,QAAgB,EAChB,QAAgB,EAChB,SAAiB,EACjB,OAAe;IAEf,MAAM,IAAI,GAAG,GAAG,QAAQ,IAAI,QAAQ,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;IAC/D,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1E,OAAO,SAAS,IAAI,EAAE,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAoB,EAAE,UAAwB,eAAe;IACrF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;QAE5C,6DAA6D;QAC7D,IAAI,aAAa,GAAG,UAAU,GAAG,OAAO,CAAC,SAAS,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9E,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,OAAO,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC;gBACzE,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,IAAI,CAAC,YAAY;gBAC3B,OAAO;gBACP,SAAS;gBACT,OAAO;gBACP,MAAM,EAAE,aAAa;aACtB,CAAC,CAAC;YAEH,uDAAuD;YACvD,MAAM,YAAY,GAAa,EAAE,CAAC;YAClC,IAAI,aAAa,GAAG,CAAC,CAAC;YAEtB,KAAK,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1F,MAAM,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,iBAAiB,GAAG,WAAW,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;gBAC1D,YAAY,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;gBAClC,aAAa,IAAI,iBAAiB,CAAC;YACrC,CAAC;YAED,gCAAgC;YAChC,YAAY,GAAG,YAAY,CAAC;YAC5B,aAAa,GAAG,aAAa,CAAC;YAC9B,SAAS;gBACP,SAAS,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC;QAC1F,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,aAAa,IAAI,UAAU,CAAC;IAC9B,CAAC;IAED,8BAA8B;IAC9B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC;YACzE,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI,CAAC,YAAY;YAC3B,OAAO;YACP,SAAS;YACT,OAAO;YACP,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CACxB,KAAuB,EACvB,UAAwB,eAAe;IAEvC,MAAM,SAAS,GAAY,EAAE,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* File Discovery Module
|
|
3
3
|
*
|
|
4
4
|
* Discovers files in sources based on include/exclude patterns.
|
|
5
|
+
* Supports incremental indexing via content hashing.
|
|
5
6
|
*/
|
|
6
7
|
import type { Source } from '../config/types.js';
|
|
7
8
|
/** File info returned by discovery */
|
|
@@ -16,7 +17,13 @@ export interface DiscoveredFile {
|
|
|
16
17
|
content: string;
|
|
17
18
|
/** File size in bytes */
|
|
18
19
|
size: number;
|
|
20
|
+
/** Content hash (SHA-256) for change detection */
|
|
21
|
+
contentHash: string;
|
|
19
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* Compute SHA-256 hash of content
|
|
25
|
+
*/
|
|
26
|
+
export declare function computeContentHash(content: string): string;
|
|
20
27
|
/** Discovery result for a source */
|
|
21
28
|
export interface DiscoveryResult {
|
|
22
29
|
sourceId: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../../src/indexer/discovery.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../../src/indexer/discovery.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAKjD,sCAAsC;AACtC,MAAM,WAAW,cAAc;IAC7B,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,mBAAmB;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,yBAAyB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,oCAAoC;AACpC,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,eAAe,CAsD9E;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,eAAe,EAAE,CAEtF"}
|
|
@@ -2,12 +2,20 @@
|
|
|
2
2
|
* File Discovery Module
|
|
3
3
|
*
|
|
4
4
|
* Discovers files in sources based on include/exclude patterns.
|
|
5
|
+
* Supports incremental indexing via content hashing.
|
|
5
6
|
*/
|
|
6
7
|
import { resolve } from 'path';
|
|
7
8
|
import { readFileSync, statSync } from 'fs';
|
|
9
|
+
import { createHash } from 'crypto';
|
|
8
10
|
import fg from 'fast-glob';
|
|
9
11
|
/** Maximum file size to process (100KB) */
|
|
10
12
|
const MAX_FILE_SIZE = 100 * 1024;
|
|
13
|
+
/**
|
|
14
|
+
* Compute SHA-256 hash of content
|
|
15
|
+
*/
|
|
16
|
+
export function computeContentHash(content) {
|
|
17
|
+
return createHash('sha256').update(content).digest('hex');
|
|
18
|
+
}
|
|
11
19
|
/**
|
|
12
20
|
* Discover all files in a source
|
|
13
21
|
*/
|
|
@@ -44,6 +52,7 @@ export function discoverFiles(source, baseDir) {
|
|
|
44
52
|
sourceId: source.id,
|
|
45
53
|
content,
|
|
46
54
|
size: stats.size,
|
|
55
|
+
contentHash: computeContentHash(content),
|
|
47
56
|
});
|
|
48
57
|
}
|
|
49
58
|
catch {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discovery.js","sourceRoot":"","sources":["../../src/indexer/discovery.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"discovery.js","sourceRoot":"","sources":["../../src/indexer/discovery.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,MAAM,WAAW,CAAC;AAG3B,2CAA2C;AAC3C,MAAM,aAAa,GAAG,GAAG,GAAG,IAAI,CAAC;AAkBjC;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5D,CAAC;AASD;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,OAAe;IAC3D,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IAEjD,sBAAsB;IACtB,MAAM,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE;QAC/C,GAAG,EAAE,UAAU;QACf,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;QAC/B,SAAS,EAAE,IAAI;QACf,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAqB,EAAE,CAAC;IACnC,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;QACnC,MAAM,YAAY,GAAG,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAEvD,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;YAErC,gCAAgC;YAChC,IAAI,KAAK,CAAC,IAAI,GAAG,aAAa,EAAE,CAAC;gBAC/B,OAAO,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YAED,oBAAoB;YACpB,MAAM,OAAO,GAAG,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YAEpD,6DAA6D;YAC7D,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,OAAO,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,YAAY;gBAClB,YAAY;gBACZ,QAAQ,EAAE,MAAM,CAAC,EAAE;gBACnB,OAAO;gBACP,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,WAAW,EAAE,kBAAkB,CAAC,OAAO,CAAC;aACzC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,2BAA2B;YAC3B,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,EAAE;QACnB,KAAK;QACL,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAiB,EAAE,OAAe;IACjE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AACjE,CAAC"}
|
package/dist/indexer/index.d.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Main Indexer Module
|
|
3
3
|
*
|
|
4
4
|
* Orchestrates file discovery, chunking, embedding, and storage.
|
|
5
|
+
* Supports incremental indexing via content hashing.
|
|
5
6
|
*/
|
|
6
7
|
import Database from 'better-sqlite3';
|
|
7
8
|
import { type ChunkOptions } from './chunker.js';
|
|
@@ -13,6 +14,12 @@ export interface IndexStats {
|
|
|
13
14
|
chunks: number;
|
|
14
15
|
skipped: number;
|
|
15
16
|
timeMs: number;
|
|
17
|
+
/** Files that were actually re-indexed (changed/new) */
|
|
18
|
+
filesChanged: number;
|
|
19
|
+
/** Files that were unchanged (skipped) */
|
|
20
|
+
filesUnchanged: number;
|
|
21
|
+
/** Files that were removed from index */
|
|
22
|
+
filesRemoved: number;
|
|
16
23
|
}
|
|
17
24
|
/** Progress update */
|
|
18
25
|
export interface IndexProgress {
|
|
@@ -23,15 +30,22 @@ export interface IndexProgress {
|
|
|
23
30
|
}
|
|
24
31
|
/** Progress callback */
|
|
25
32
|
export type IndexProgressCallback = (progress: IndexProgress) => void;
|
|
33
|
+
/** Options for indexing */
|
|
34
|
+
export interface IndexOptions extends ChunkOptions {
|
|
35
|
+
/** Force full re-index (ignore hashes) */
|
|
36
|
+
force?: boolean;
|
|
37
|
+
}
|
|
26
38
|
/**
|
|
27
|
-
* Index all sources
|
|
39
|
+
* Index all sources with incremental support
|
|
28
40
|
*/
|
|
29
|
-
export declare function indexSources(sources: Source[], baseDir: string, db: Database.Database, chunkOptions: ChunkOptions, onProgress?: IndexProgressCallback
|
|
41
|
+
export declare function indexSources(sources: Source[], baseDir: string, db: Database.Database, chunkOptions: ChunkOptions, onProgress?: IndexProgressCallback, options?: {
|
|
42
|
+
force?: boolean;
|
|
43
|
+
}): Promise<IndexStats>;
|
|
30
44
|
/**
|
|
31
45
|
* Read embedding from blob
|
|
32
46
|
*/
|
|
33
47
|
export declare function readEmbedding(blob: Buffer): number[];
|
|
34
|
-
export { discoverFiles, type DiscoveredFile, type DiscoveryResult } from './discovery.js';
|
|
48
|
+
export { discoverFiles, computeContentHash, type DiscoveredFile, type DiscoveryResult } from './discovery.js';
|
|
35
49
|
export { chunkFiles, chunkFile, countTokens, type Chunk, type ChunkOptions } from './chunker.js';
|
|
36
50
|
export { embed, embedBatch, embedChunks, cosineSimilarity, EMBEDDING_DIM, type EmbeddedChunk, } from './embeddings.js';
|
|
37
51
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/indexer/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/indexer/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAGtC,OAAO,EAAc,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAE7D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,0BAA0B;AAC1B,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,cAAc,EAAE,MAAM,CAAC;IACvB,yCAAyC;IACzC,YAAY,EAAE,MAAM,CAAC;CACtB;AAkED,sBAAsB;AACtB,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,WAAW,GAAG,UAAU,GAAG,WAAW,GAAG,SAAS,CAAC;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAwB;AACxB,MAAM,MAAM,qBAAqB,GAAG,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;AAEtE,2BAA2B;AAC3B,MAAM,WAAW,YAAa,SAAQ,YAAY;IAChD,0CAA0C;IAC1C,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE,MAAM,EACf,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,YAAY,EAAE,YAAY,EAC1B,UAAU,CAAC,EAAE,qBAAqB,EAClC,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,GAC5B,OAAO,CAAC,UAAU,CAAC,CA6GrB;AAgHD;;GAEG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAGpD;AAGD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,KAAK,cAAc,EAAE,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC9G,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,KAAK,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AACjG,OAAO,EACL,KAAK,EACL,UAAU,EACV,WAAW,EACX,gBAAgB,EAChB,aAAa,EACb,KAAK,aAAa,GACnB,MAAM,iBAAiB,CAAC"}
|