@modular-prompt/extract 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -3
- package/dist/cache-lifecycle.d.ts +2 -0
- package/dist/cache-lifecycle.d.ts.map +1 -1
- package/dist/cache-lifecycle.js +1 -0
- package/dist/cache-lifecycle.js.map +1 -1
- package/dist/cli/add-command.d.ts +2 -0
- package/dist/cli/add-command.d.ts.map +1 -1
- package/dist/cli/add-command.js +7 -2
- package/dist/cli/add-command.js.map +1 -1
- package/dist/cli/args.d.ts +1 -0
- package/dist/cli/args.d.ts.map +1 -1
- package/dist/cli/args.js +15 -0
- package/dist/cli/args.js.map +1 -1
- package/dist/cli/constants.d.ts +4 -0
- package/dist/cli/constants.d.ts.map +1 -1
- package/dist/cli/constants.js +19 -0
- package/dist/cli/constants.js.map +1 -1
- package/dist/cli/extract-command.d.ts +2 -0
- package/dist/cli/extract-command.d.ts.map +1 -1
- package/dist/cli/extract-command.js +14 -3
- package/dist/cli/extract-command.js.map +1 -1
- package/dist/cli/list-command.d.ts.map +1 -1
- package/dist/cli/list-command.js +3 -6
- package/dist/cli/list-command.js.map +1 -1
- package/dist/cli/store.d.ts +21 -1
- package/dist/cli/store.d.ts.map +1 -1
- package/dist/cli/store.js +189 -4
- package/dist/cli/store.js.map +1 -1
- package/dist/cli.js +6 -2
- package/dist/cli.js.map +1 -1
- package/dist/create-extract-session.d.ts.map +1 -1
- package/dist/create-extract-session.js +4 -1
- package/dist/create-extract-session.js.map +1 -1
- package/dist/extract-store.d.ts +27 -0
- package/dist/extract-store.d.ts.map +1 -1
- package/dist/extract-store.js +61 -2
- package/dist/extract-store.js.map +1 -1
- package/dist/types.d.ts +6 -0
- package/dist/types.d.ts.map +1 -1
- package/docs/API.md +8 -4
- package/docs/CACHE_DESIGN.md +89 -7
- package/package.json +3 -3
package/dist/cli/store.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { stat } from 'node:fs/promises';
|
|
2
|
-
import { join } from 'node:path';
|
|
1
|
+
import { readdir, readFile, stat } from 'node:fs/promises';
|
|
2
|
+
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
3
3
|
const STORE_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/;
|
|
4
4
|
const RESERVED_STORE_NAMES = new Set(['create', 'add', 'extract', 'list', 'clean']);
|
|
5
5
|
/** Validate a store name used as a single directory component. */
|
|
@@ -29,8 +29,193 @@ export async function storeExists(storeDir) {
|
|
|
29
29
|
throw error;
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
|
-
/** Return whether
|
|
32
|
+
/** Return whether a store entry is a persisted KV cache file or namespace. */
|
|
33
33
|
export function isKvCacheFile(filename) {
|
|
34
|
-
return filename.endsWith('.safetensors.zip')
|
|
34
|
+
return filename.endsWith('.safetensors.zip')
|
|
35
|
+
|| filename.endsWith('.pytorch-cache')
|
|
36
|
+
|| filename.endsWith('.vlm.safetensors')
|
|
37
|
+
|| filename.endsWith('.vlm-vision.safetensors');
|
|
38
|
+
}
|
|
39
|
+
const CACHE_INDEX_FILENAME = 'cache-index.json';
|
|
40
|
+
function isRecord(value) {
|
|
41
|
+
return typeof value === 'object' && value !== null;
|
|
42
|
+
}
|
|
43
|
+
function isVlmCachePath(cachePath) {
|
|
44
|
+
return cachePath.split(sep).some((segment) => segment.endsWith('.vlm.safetensors')
|
|
45
|
+
|| segment.endsWith('.vlm-vision.safetensors'));
|
|
46
|
+
}
|
|
47
|
+
async function hasHealthyCacheFile(cachePath) {
|
|
48
|
+
let cacheStat;
|
|
49
|
+
try {
|
|
50
|
+
cacheStat = await stat(cachePath);
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
if (cacheStat.isDirectory()) {
|
|
56
|
+
let entries;
|
|
57
|
+
try {
|
|
58
|
+
entries = await readdir(cachePath, { withFileTypes: true });
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
for (const entry of entries) {
|
|
64
|
+
const childPath = join(cachePath, entry.name);
|
|
65
|
+
if (entry.isDirectory() && await hasHealthyCacheFile(childPath)) {
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
if (entry.isFile() && entry.name.endsWith('.safetensors')) {
|
|
69
|
+
if (await hasHealthyCacheFile(childPath)) {
|
|
70
|
+
return true;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
if (!cacheStat.isFile()) {
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
try {
|
|
80
|
+
const metadata = JSON.parse(await readFile(`${cachePath}.meta.json`, 'utf-8'));
|
|
81
|
+
return typeof metadata.token_count === 'number' && metadata.token_count > 0;
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
async function readCacheIndex(storeDir) {
|
|
88
|
+
try {
|
|
89
|
+
const parsed = JSON.parse(await readFile(join(storeDir, CACHE_INDEX_FILENAME), 'utf-8'));
|
|
90
|
+
if (!isRecord(parsed)
|
|
91
|
+
|| parsed.version !== 1
|
|
92
|
+
|| !Array.isArray(parsed.entries)
|
|
93
|
+
|| !parsed.entries.every(isRecord)) {
|
|
94
|
+
return { present: true };
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
present: true,
|
|
98
|
+
entries: parsed.entries,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
|
|
103
|
+
return { present: false };
|
|
104
|
+
}
|
|
105
|
+
return { present: true };
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
function resolveIndexedCachePath(storeDir, entry) {
|
|
109
|
+
if (typeof entry.key !== 'string' || entry.key.length === 0) {
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
let indexedPath;
|
|
113
|
+
if (typeof entry.path === 'string') {
|
|
114
|
+
if (entry.path.startsWith('memory://')) {
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
indexedPath = entry.path;
|
|
118
|
+
}
|
|
119
|
+
else if (entry.backend === 'pytorch') {
|
|
120
|
+
indexedPath = `${entry.key}.pytorch-cache`;
|
|
121
|
+
}
|
|
122
|
+
else if (entry.backend === undefined || entry.backend === 'lm') {
|
|
123
|
+
indexedPath = `${entry.key}.safetensors.zip`;
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
return undefined;
|
|
127
|
+
}
|
|
128
|
+
const resolvedStoreDir = resolve(storeDir);
|
|
129
|
+
const resolvedPath = isAbsolute(indexedPath)
|
|
130
|
+
? resolve(indexedPath)
|
|
131
|
+
: resolve(resolvedStoreDir, indexedPath);
|
|
132
|
+
const pathFromStore = relative(resolvedStoreDir, resolvedPath);
|
|
133
|
+
if (pathFromStore.length === 0
|
|
134
|
+
|| pathFromStore === '..'
|
|
135
|
+
|| pathFromStore.startsWith(`..${sep}`)) {
|
|
136
|
+
return undefined;
|
|
137
|
+
}
|
|
138
|
+
return resolvedPath;
|
|
139
|
+
}
|
|
140
|
+
async function listRawCachePaths(storeDir) {
|
|
141
|
+
let entries;
|
|
142
|
+
try {
|
|
143
|
+
entries = await readdir(storeDir, { withFileTypes: true });
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
return [];
|
|
147
|
+
}
|
|
148
|
+
return entries
|
|
149
|
+
.filter((entry) => (entry.isFile() || entry.isDirectory()) && isKvCacheFile(entry.name))
|
|
150
|
+
.map((entry) => join(storeDir, entry.name));
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Inspect the persisted cache without starting a model runtime.
|
|
154
|
+
*
|
|
155
|
+
* A cache-index entry is considered usable only when its referenced cache and
|
|
156
|
+
* token metadata exist. A store without an index can still be used for an
|
|
157
|
+
* exact cache hit (legacy layout), but it cannot provide an incremental base.
|
|
158
|
+
*/
|
|
159
|
+
export async function inspectStoreKvCache(storeDir) {
|
|
160
|
+
const rawCachePaths = await listRawCachePaths(storeDir);
|
|
161
|
+
const index = await readCacheIndex(storeDir);
|
|
162
|
+
if (!index.present) {
|
|
163
|
+
const hasHealthyCache = (await Promise.all(rawCachePaths.map((cachePath) => hasHealthyCacheFile(cachePath)))).some(Boolean);
|
|
164
|
+
return {
|
|
165
|
+
hasKvCache: hasHealthyCache,
|
|
166
|
+
needsRebuild: !hasHealthyCache,
|
|
167
|
+
hasIncrementalBase: false,
|
|
168
|
+
...(hasHealthyCache
|
|
169
|
+
? {}
|
|
170
|
+
: { issue: rawCachePaths.length > 0 ? 'cache-invalid' : 'missing' }),
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
if (!index.entries) {
|
|
174
|
+
return {
|
|
175
|
+
hasKvCache: false,
|
|
176
|
+
needsRebuild: true,
|
|
177
|
+
hasIncrementalBase: false,
|
|
178
|
+
issue: 'index-invalid',
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
const activeEntries = index.entries.filter((entry) => entry.hint !== 'release');
|
|
182
|
+
if (activeEntries.length === 0) {
|
|
183
|
+
return {
|
|
184
|
+
hasKvCache: false,
|
|
185
|
+
needsRebuild: true,
|
|
186
|
+
hasIncrementalBase: false,
|
|
187
|
+
issue: 'index-entry-missing',
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
const indexedCachePaths = activeEntries.map((entry) => resolveIndexedCachePath(storeDir, entry));
|
|
191
|
+
const entriesAreHealthy = await Promise.all(indexedCachePaths.map(async (cachePath) => {
|
|
192
|
+
return cachePath !== undefined && await hasHealthyCacheFile(cachePath);
|
|
193
|
+
}));
|
|
194
|
+
if (!entriesAreHealthy.every(Boolean)) {
|
|
195
|
+
return {
|
|
196
|
+
hasKvCache: false,
|
|
197
|
+
needsRebuild: true,
|
|
198
|
+
hasIncrementalBase: false,
|
|
199
|
+
issue: 'index-entry-missing',
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
// VLM exact snapshots are valid exact-cache hits, but the VLM controllers
|
|
203
|
+
// intentionally do not search them as incremental bases. `add` must
|
|
204
|
+
// therefore take the full-rebuild path even when the snapshot is healthy.
|
|
205
|
+
const hasVlmNamespace = activeEntries.some((entry, index) => {
|
|
206
|
+
const indexedPath = indexedCachePaths[index];
|
|
207
|
+
return entry.backend === 'vlm'
|
|
208
|
+
|| (indexedPath !== undefined && isVlmCachePath(indexedPath));
|
|
209
|
+
});
|
|
210
|
+
return {
|
|
211
|
+
hasKvCache: true,
|
|
212
|
+
needsRebuild: false,
|
|
213
|
+
hasIncrementalBase: !hasVlmNamespace,
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
/** Warning emitted after an automatic full cache rebuild. */
|
|
217
|
+
export function formatKvCacheRebuildWarning(storename, materialCount) {
|
|
218
|
+
return `warning: KV cache missing for store '${storename}'; `
|
|
219
|
+
+ `rebuilt cache from manifest (${materialCount} materials).`;
|
|
35
220
|
}
|
|
36
221
|
//# sourceMappingURL=store.js.map
|
package/dist/cli/store.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/cli/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/cli/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAErE,MAAM,kBAAkB,GAAG,6BAA6B,CAAC;AACzD,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAEpF,kEAAkE;AAClE,MAAM,UAAU,iBAAiB,CAAC,SAAiB;IACjD,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CACb,sBAAsB,MAAM,CAAC,SAAS,CAAC,yCAAyC,CACjF,CAAC;IACJ,CAAC;IAED,IAAI,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,sBAAsB,SAAS,kBAAkB,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAAC,YAAoB,EAAE,SAAiB;IACrE,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC7B,OAAO,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;AACvC,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAgB;IAChD,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,OAAO,QAAQ,CAAC,QAAQ,CAAC,kBAAkB,CAAC;WACvC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC;WACnC,QAAQ,CAAC,QAAQ,CAAC,kBAAkB,CAAC;WACrC,QAAQ,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,oBAAoB,GAAG,kBAAkB,CAAC;AAyBhD,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC;AACrD,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB;IACvC,OAAO,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAC3C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;WACjC,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,SAAiB;IAClD,IAAI,SAAS,CAAC;IACd,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC;QAC5B,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9C,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC;gBAChE,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC1D,IAAI,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC;oBACzC,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,GAAG,SAAS,YAAY,EAAE,OAAO,CAAC,CAE5E,CAAC;QACF,OAAO,OAAO,QAAQ,CAAC,WAAW,KAAK,QAAQ,IAAI,QAAQ,CAAC,WAAW,GAAG,CAAC,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,oBAAoB,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QAClG,IACE,CAAC,QAAQ,CAAC,MAAM,CAAC;eACd,MAAM,CAAC,OAAO,KAAK,CAAC;eACpB,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;eAC9B,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAClC,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3B,CAAC;QACD,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,MAAM,CAAC,OAA4B;SAC7C,CAAC;IACJ,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAC9B,QAAgB,EAChB,KAAsB;IAEtB,IAAI,OAAO,KAAK,CAAC,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5D,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,WAAmB,CAAC;IACxB,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACnC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YACvC,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;IAC3B,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACvC,WAAW,GAAG,GAAG,KAAK,CAAC,GAAG,gBAAgB,CAAC;IAC7C,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACjE,WAAW,GAAG,GAAG,KAAK,CAAC,GAAG,kBAAkB,CAAC;IAC/C,CAAC;SAAM,CAAC;QACN,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,YAAY,GAAG,UAAU,CAAC,WAAW,CAAC;QAC1C,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;QACtB,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;IAC3C,MAAM,aAAa,GAAG,QAAQ,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;IAC/D,IACE,aAAa,CAAC,MAAM,KAAK,CAAC;WACvB,aAAa,KAAK,IAAI;WACtB,aAAa,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,EACvC,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IAC/C,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SACvF,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAgB;IACxD,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;IAE7C,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,eAAe,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CACxC,aAAa,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CACjE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjB,OAAO;YACL,UAAU,EAAE,eAAe;YAC3B,YAAY,EAAE,CAAC,eAAe;YAC9B,kBAAkB,EAAE,KAAK;YACzB,GAAG,CAAC,eAAe;gBACjB,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAwB,CAAC,CAAC,CAAC,SAAkB,EAAE,CAAC;SACzF,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO;YACL,UAAU,EAAE,KAAK;YACjB,YAAY,EAAE,IAAI;YAClB,kBAAkB,EAAE,KAAK;YACzB,KAAK,EAAE,eAAe;SACvB,CAAC;IACJ,CAAC;IAED,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAChF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO;YACL,UAAU,EAAE,KAAK;YACjB,YAAY,EAAE,IAAI;YAClB,kBAAkB,EAAE,KAAK;YACzB,KAAK,EAAE,qBAAqB;SAC7B,CAAC;IACJ,CAAC;IAED,MAAM,iBAAiB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,uBAAuB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;IACjG,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;QACpF,OAAO,SAAS,KAAK,SAAS,IAAI,MAAM,mBAAmB,CAAC,SAAS,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC,CAAC;IACJ,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,OAAO;YACL,UAAU,EAAE,KAAK;YACjB,YAAY,EAAE,IAAI;YAClB,kBAAkB,EAAE,KAAK;YACzB,KAAK,EAAE,qBAAqB;SAC7B,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,qEAAqE;IACrE,0EAA0E;IAC1E,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC1D,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,CAAC,OAAO,KAAK,KAAK;eACzB,CAAC,WAAW,KAAK,SAAS,IAAI,cAAc,CAAC,WAAW,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,UAAU,EAAE,IAAI;QAChB,YAAY,EAAE,KAAK;QACnB,kBAAkB,EAAE,CAAC,eAAe;KACrC,CAAC;AACJ,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,2BAA2B,CAAC,SAAiB,EAAE,aAAqB;IAClF,OAAO,wCAAwC,SAAS,KAAK;UACzD,gCAAgC,aAAa,cAAc,CAAC;AAClE,CAAC"}
|
package/dist/cli.js
CHANGED
|
@@ -16,8 +16,8 @@ function printHelp() {
|
|
|
16
16
|
|
|
17
17
|
Usage:
|
|
18
18
|
modular-prompt-extract create <storename> [-d <cache-dir>] [-m <model>] [--provider <mlx|pytorch>] [--dry-run] <files...>
|
|
19
|
-
modular-prompt-extract add <storename> [-d <cache-dir>] [--dry-run] <files...>
|
|
20
|
-
modular-prompt-extract extract <storename> [-d <cache-dir>] [--max-tokens <n>] [--dry-run] <query...>
|
|
19
|
+
modular-prompt-extract add <storename> [-d <cache-dir>] [--auto-rebuild-cache|--no-auto-rebuild-cache] [--dry-run] <files...>
|
|
20
|
+
modular-prompt-extract extract <storename> [-d <cache-dir>] [--max-tokens <n>] [--auto-rebuild-cache|--no-auto-rebuild-cache] [--dry-run] <query...>
|
|
21
21
|
modular-prompt-extract list [-d <cache-dir>]
|
|
22
22
|
modular-prompt-extract clean <storename> [-d <cache-dir>]
|
|
23
23
|
modular-prompt-extract clean --all [-d <cache-dir>]
|
|
@@ -34,6 +34,8 @@ Options:
|
|
|
34
34
|
-m, --model <model> Model alias from models.yaml or raw model id
|
|
35
35
|
--provider <provider> Extract provider for create (mlx or pytorch)
|
|
36
36
|
--max-tokens <n> Max tokens for extract (default: ${DEFAULT_MAX_TOKENS})
|
|
37
|
+
--auto-rebuild-cache Enable KV cache regeneration (add/extract; default)
|
|
38
|
+
--no-auto-rebuild-cache Do not regenerate a missing KV cache (add/extract)
|
|
37
39
|
--dry-run Compile and print full prompt text (no driver / no cache write)
|
|
38
40
|
--all Remove the entire cache container (clean only)
|
|
39
41
|
-h, --help Show help
|
|
@@ -81,6 +83,7 @@ async function main() {
|
|
|
81
83
|
storename: parsed.storename,
|
|
82
84
|
files: parsed.positional,
|
|
83
85
|
dryRun: parsed.dryRun,
|
|
86
|
+
autoRebuildCache: parsed.autoRebuildCache,
|
|
84
87
|
});
|
|
85
88
|
if (typeof output === 'string') {
|
|
86
89
|
process.stdout.write(`${output}\n`);
|
|
@@ -94,6 +97,7 @@ async function main() {
|
|
|
94
97
|
query: parsed.positional.join(' '),
|
|
95
98
|
maxTokens: parsed.maxTokens,
|
|
96
99
|
dryRun: parsed.dryRun,
|
|
100
|
+
autoRebuildCache: parsed.autoRebuildCache,
|
|
97
101
|
});
|
|
98
102
|
process.stdout.write(`${text}\n`);
|
|
99
103
|
return;
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AACpF,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAC5B,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,OAAO,CAAC,CACnC,CAAC;AAEzB,SAAS,SAAS;IAChB,OAAO,CAAC,GAAG,CAAC,2BAA2B,WAAW,CAAC,OAAO;;;;;;;;;;;;;;;;;;iEAkBK,0BAA0B,EAAE;;;8DAG/B,kBAAkB
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AACpF,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAC5B,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,OAAO,CAAC,CACnC,CAAC;AAEzB,SAAS,SAAS;IAChB,OAAO,CAAC,GAAG,CAAC,2BAA2B,WAAW,CAAC,OAAO;;;;;;;;;;;;;;;;;;iEAkBK,0BAA0B,EAAE;;;8DAG/B,kBAAkB;;;;;;;;;;;;;;;;;;CAkB/E,CAAC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhD,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QACjD,SAAS,EAAE,CAAC;QACZ,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,0BAA0B,EAAE,CAAC;IAEjE,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC;YACpC,QAAQ;YACR,SAAS,EAAE,MAAM,CAAC,SAAU;YAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,KAAK,EAAE,MAAM,CAAC,UAAU;YACxB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QACH,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC;QACtC,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YACjC,QAAQ;YACR,SAAS,EAAE,MAAM,CAAC,SAAU;YAC5B,KAAK,EAAE,MAAM,CAAC,UAAU;YACxB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC;QACtC,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC;YACnC,QAAQ;YACR,SAAS,EAAE,MAAM,CAAC,SAAU;YAC5B,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;YAClC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC;YACpC,QAAQ;YACR,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QACrC,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC;QAChC,QAAQ;KACT,CAAC,CAAC;IACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;AACpC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC9B,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-extract-session.d.ts","sourceRoot":"","sources":["../src/create-extract-session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAiC,cAAc,EAA8B,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAyBnI,wBAAgB,oBAAoB,CAAC,QAAQ,GAAG,cAAc,EAC5D,OAAO,EAAE,qBAAqB,CAAC,QAAQ,CAAC,GACvC,cAAc,
|
|
1
|
+
{"version":3,"file":"create-extract-session.d.ts","sourceRoot":"","sources":["../src/create-extract-session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAiC,cAAc,EAA8B,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAyBnI,wBAAgB,oBAAoB,CAAC,QAAQ,GAAG,cAAc,EAC5D,OAAO,EAAE,qBAAqB,CAAC,QAAQ,CAAC,GACvC,cAAc,CAiFhB"}
|
|
@@ -29,6 +29,7 @@ export function createExtractSession(options) {
|
|
|
29
29
|
await ensureCacheControllerReady(driver, cacheState);
|
|
30
30
|
await prepareSessionCache(cacheController, model, sessionBaseModule, corpus, request, baseModule, cacheState, {
|
|
31
31
|
required: options.cachePreparation === 'required',
|
|
32
|
+
...(options.autoRebuildCache === false ? { readOnly: true } : {}),
|
|
32
33
|
maxImageSize: options.maxImageSize,
|
|
33
34
|
});
|
|
34
35
|
const compiled = compileExtractPrompt(sessionBaseModule, corpus, request, baseModule);
|
|
@@ -38,7 +39,9 @@ export function createExtractSession(options) {
|
|
|
38
39
|
cache: false,
|
|
39
40
|
cacheHandle: cacheState.handle,
|
|
40
41
|
}
|
|
41
|
-
:
|
|
42
|
+
: options.autoRebuildCache === false
|
|
43
|
+
? { ...request.options, cache: 'read-only' }
|
|
44
|
+
: request.options;
|
|
42
45
|
const queryResult = await driver.query(compiled, queryOptions);
|
|
43
46
|
const result = {
|
|
44
47
|
text: queryResult.content,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-extract-session.js","sourceRoot":"","sources":["../src/create-extract-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAI7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EACL,0BAA0B,EAC1B,mBAAmB,EACnB,mBAAmB,GAEpB,MAAM,sBAAsB,CAAC;AAE9B,SAAS,sBAAsB,CAC7B,UAAkC,EAClC,MAAe;IAEf,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,MAAM,YAAY,GAAiB;QACjC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;KAC5C,CAAC;IAEF,OAAO,KAAK,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,OAAwC;IAExC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAE5B,MAAM,iBAAiB,GAAG,sBAAsB,CAC9C,qBAAqB,CAAC,OAAO,CAAC,EAC9B,MAAM,CACP,CAAC;IACF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,UAAU,GAAwB;QACtC,MAAM,EAAE,IAAI;QACZ,eAAe,EAAE,KAAK;KACvB,CAAC;IACF,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,OAAO;QACL,KAAK,CAAC,OAAO,CAAC,OAAuB;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAC9C,CAAC;YAED,MAAM,0BAA0B,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YACrD,MAAM,mBAAmB,CACvB,eAAe,EACf,KAAK,EACL,iBAAiB,EACjB,MAAM,EACN,OAAO,EACP,UAAU,EACV,UAAU,EACV;gBACE,QAAQ,EAAE,OAAO,CAAC,gBAAgB,KAAK,UAAU;gBACjD,YAAY,EAAE,OAAO,CAAC,YAAY;aACnC,CACF,CAAC;YAEF,MAAM,QAAQ,GAAG,oBAAoB,CACnC,iBAAiB,EACjB,MAAM,EACN,OAAO,EACP,UAAU,CACX,CAAC;YACF,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM;gBACpC,CAAC,CAAC;oBACE,GAAG,OAAO,CAAC,OAAO;oBAClB,KAAK,EAAE,KAAc;oBACrB,WAAW,EAAE,UAAU,CAAC,MAAM;iBAC/B;gBACH,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"create-extract-session.js","sourceRoot":"","sources":["../src/create-extract-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAI7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EACL,0BAA0B,EAC1B,mBAAmB,EACnB,mBAAmB,GAEpB,MAAM,sBAAsB,CAAC;AAE9B,SAAS,sBAAsB,CAC7B,UAAkC,EAClC,MAAe;IAEf,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,MAAM,YAAY,GAAiB;QACjC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;KAC5C,CAAC;IAEF,OAAO,KAAK,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,OAAwC;IAExC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAE5B,MAAM,iBAAiB,GAAG,sBAAsB,CAC9C,qBAAqB,CAAC,OAAO,CAAC,EAC9B,MAAM,CACP,CAAC;IACF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,UAAU,GAAwB;QACtC,MAAM,EAAE,IAAI;QACZ,eAAe,EAAE,KAAK;KACvB,CAAC;IACF,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,OAAO;QACL,KAAK,CAAC,OAAO,CAAC,OAAuB;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAC9C,CAAC;YAED,MAAM,0BAA0B,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YACrD,MAAM,mBAAmB,CACvB,eAAe,EACf,KAAK,EACL,iBAAiB,EACjB,MAAM,EACN,OAAO,EACP,UAAU,EACV,UAAU,EACV;gBACE,QAAQ,EAAE,OAAO,CAAC,gBAAgB,KAAK,UAAU;gBACjD,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjE,YAAY,EAAE,OAAO,CAAC,YAAY;aACnC,CACF,CAAC;YAEF,MAAM,QAAQ,GAAG,oBAAoB,CACnC,iBAAiB,EACjB,MAAM,EACN,OAAO,EACP,UAAU,CACX,CAAC;YACF,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM;gBACpC,CAAC,CAAC;oBACE,GAAG,OAAO,CAAC,OAAO;oBAClB,KAAK,EAAE,KAAc;oBACrB,WAAW,EAAE,UAAU,CAAC,MAAM;iBAC/B;gBACH,CAAC,CAAC,OAAO,CAAC,gBAAgB,KAAK,KAAK;oBAClC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,WAAoB,EAAE;oBACrD,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;YAEtB,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YAE/D,MAAM,MAAM,GAAkB;gBAC5B,IAAI,EAAE,WAAW,CAAC,OAAO;gBACzB,UAAU,EAAE,WAAW,CAAC,gBAAgB;gBACxC,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,KAAK,EAAE,OAAO,CAAC,MAAM;aACtB,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,UAAU;YACR,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,OAAoC;YAC9C,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YACD,MAAM,GAAG,IAAI,CAAC;YAEd,IAAI,OAAO,EAAE,YAAY,KAAK,KAAK,EAAE,CAAC;gBACpC,mBAAmB,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/extract-store.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { MlxBackendMode } from '@modular-prompt/driver';
|
|
|
2
2
|
import type { MaterialInput } from './extract-elements.js';
|
|
3
3
|
import type { ExtractProvider, ExtractRuntime } from './extract-runtime-types.js';
|
|
4
4
|
import { type ExtractCacheManifest } from './cli/manifest.js';
|
|
5
|
+
import { type StoreKvCacheStatus } from './cli/store.js';
|
|
5
6
|
export interface PrepareExtractCacheOptions {
|
|
6
7
|
/** Directory containing the persistent cache files. */
|
|
7
8
|
cacheDir: string;
|
|
@@ -41,6 +42,28 @@ export declare function prepareExtractCache(options: PrepareExtractCacheOptions)
|
|
|
41
42
|
export declare function mergeMaterials(existing: readonly MaterialInput[], incoming: readonly MaterialInput[]): MaterialInput[];
|
|
42
43
|
/** Read and validate a named store manifest with a user-facing missing-store error. */
|
|
43
44
|
export declare function readExtractStoreManifest(storeDir: string, storename: string): Promise<ExtractCacheManifest>;
|
|
45
|
+
export interface EnsureStoreKvCacheOptions {
|
|
46
|
+
/** Existing named store directory. */
|
|
47
|
+
storeDir: string;
|
|
48
|
+
/** Store name used in actionable error messages. */
|
|
49
|
+
storename: string;
|
|
50
|
+
/** Manifest already read by the caller, when available. */
|
|
51
|
+
manifest?: ExtractCacheManifest;
|
|
52
|
+
/** Automatically recreate a missing/inconsistent cache. Defaults to true. */
|
|
53
|
+
autoRebuildCache?: boolean;
|
|
54
|
+
}
|
|
55
|
+
export interface EnsureStoreKvCacheResult {
|
|
56
|
+
status: StoreKvCacheStatus;
|
|
57
|
+
rebuilt: boolean;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Ensure that a named store has a usable persistent cache before extraction.
|
|
61
|
+
*
|
|
62
|
+
* The operation intentionally does not modify the manifest. The manifest is
|
|
63
|
+
* the corpus source of truth and a successful prefill only restores its
|
|
64
|
+
* transparent cache layer.
|
|
65
|
+
*/
|
|
66
|
+
export declare function ensureStoreKvCache(options: EnsureStoreKvCacheOptions): Promise<EnsureStoreKvCacheResult>;
|
|
44
67
|
export interface AppendToExtractStoreOptions {
|
|
45
68
|
/** Existing named store directory. */
|
|
46
69
|
storeDir: string;
|
|
@@ -52,6 +75,8 @@ export interface AppendToExtractStoreOptions {
|
|
|
52
75
|
existingManifest?: ExtractCacheManifest;
|
|
53
76
|
/** Injectable clock for deterministic callers and tests. */
|
|
54
77
|
now?: () => string;
|
|
78
|
+
/** Automatically rebuild a missing incremental base. Defaults to true. */
|
|
79
|
+
autoRebuildCache?: boolean;
|
|
55
80
|
}
|
|
56
81
|
export interface AppendToExtractStoreResult {
|
|
57
82
|
manifest: ExtractCacheManifest;
|
|
@@ -59,6 +84,8 @@ export interface AppendToExtractStoreResult {
|
|
|
59
84
|
provider: ExtractProvider;
|
|
60
85
|
backend?: MlxBackendMode;
|
|
61
86
|
addedMaterials: number;
|
|
87
|
+
/** Whether the append used a full prefill because no incremental base existed. */
|
|
88
|
+
cacheRebuilt: boolean;
|
|
62
89
|
}
|
|
63
90
|
export declare function assertExtractRuntimeMatchesManifest(runtime: Pick<ExtractRuntime, 'model' | 'provider'>, manifest: ExtractCacheManifest): void;
|
|
64
91
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-store.d.ts","sourceRoot":"","sources":["../src/extract-store.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAElF,OAAO,EAKL,KAAK,oBAAoB,EAC1B,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"extract-store.d.ts","sourceRoot":"","sources":["../src/extract-store.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAElF,OAAO,EAKL,KAAK,oBAAoB,EAC1B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAIL,KAAK,kBAAkB,EACxB,MAAM,gBAAgB,CAAC;AAExB,MAAM,WAAW,0BAA0B;IACzC,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,QAAQ,CAAC,EAAE,eAAe,CAAC;IAC3B,+EAA+E;IAC/E,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wCAAwC;IACxC,SAAS,EAAE,SAAS,aAAa,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,sEAAsE;IACtE,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAeD;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,0BAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAuC/B;AAYD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,SAAS,aAAa,EAAE,EAClC,QAAQ,EAAE,SAAS,aAAa,EAAE,GACjC,aAAa,EAAE,CA6BjB;AASD,uFAAuF;AACvF,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,oBAAoB,CAAC,CAS/B;AAED,MAAM,WAAW,yBAAyB;IACxC,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,6EAA6E;IAC7E,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,wBAAwB,CAAC,CAmCnC;AAED,MAAM,WAAW,2BAA2B;IAC1C,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,iBAAiB,EAAE,SAAS,aAAa,EAAE,CAAC;IAC5C,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;IACxC,4DAA4D;IAC5D,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,kFAAkF;IAClF,YAAY,EAAE,OAAO,CAAC;CACvB;AAmBD,wBAAgB,mCAAmC,CACjD,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE,OAAO,GAAG,UAAU,CAAC,EACnD,QAAQ,EAAE,oBAAoB,GAC7B,IAAI,CAcN;AA+BD;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,0BAA0B,CAAC,CA+DrC"}
|
package/dist/extract-store.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
-
import { cp, mkdtemp, rename, rm } from 'node:fs/promises';
|
|
2
|
+
import { cp, mkdtemp, readdir, rename, rm } from 'node:fs/promises';
|
|
3
3
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
4
4
|
import { isDeepStrictEqual } from 'node:util';
|
|
5
5
|
import { createExtractSession } from './create-extract-session.js';
|
|
6
6
|
import { createExtractRuntime } from './create-extract-runtime.js';
|
|
7
7
|
import { CACHE_PREPARE_CUE } from './cli/constants.js';
|
|
8
8
|
import { getManifestProvider, manifestExists, readManifest, writeManifest, } from './cli/manifest.js';
|
|
9
|
-
import { storeExists } from './cli/store.js';
|
|
9
|
+
import { inspectStoreKvCache, isKvCacheFile, storeExists, } from './cli/store.js';
|
|
10
10
|
async function closeRuntimePreservingError(runtime, operationError) {
|
|
11
11
|
try {
|
|
12
12
|
await runtime.close();
|
|
@@ -118,6 +118,59 @@ export async function readExtractStoreManifest(storeDir, storename) {
|
|
|
118
118
|
}
|
|
119
119
|
return readManifest(resolvedStoreDir);
|
|
120
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* Ensure that a named store has a usable persistent cache before extraction.
|
|
123
|
+
*
|
|
124
|
+
* The operation intentionally does not modify the manifest. The manifest is
|
|
125
|
+
* the corpus source of truth and a successful prefill only restores its
|
|
126
|
+
* transparent cache layer.
|
|
127
|
+
*/
|
|
128
|
+
export async function ensureStoreKvCache(options) {
|
|
129
|
+
const storeDir = resolve(options.storeDir);
|
|
130
|
+
const status = await inspectStoreKvCache(storeDir);
|
|
131
|
+
if (!status.needsRebuild || options.autoRebuildCache === false) {
|
|
132
|
+
return { status, rebuilt: false };
|
|
133
|
+
}
|
|
134
|
+
const manifest = options.manifest
|
|
135
|
+
?? await readExtractStoreManifest(storeDir, options.storename);
|
|
136
|
+
const stagingDir = await mkdtemp(join(dirname(storeDir), `.${basename(storeDir)}.rebuild-`));
|
|
137
|
+
let committed = false;
|
|
138
|
+
try {
|
|
139
|
+
await cp(storeDir, stagingDir, { recursive: true, force: true });
|
|
140
|
+
const stagedEntries = await readdir(stagingDir, { withFileTypes: true });
|
|
141
|
+
await Promise.all(stagedEntries
|
|
142
|
+
.filter((entry) => entry.name === 'cache-index.json' || isKvCacheFile(entry.name))
|
|
143
|
+
.map((entry) => rm(join(stagingDir, entry.name), { recursive: true, force: true })));
|
|
144
|
+
const prepared = await prepareExtractCache({
|
|
145
|
+
cacheDir: stagingDir,
|
|
146
|
+
model: manifest.model,
|
|
147
|
+
provider: getManifestProvider(manifest),
|
|
148
|
+
...(manifest.backend !== undefined ? { backend: manifest.backend } : {}),
|
|
149
|
+
...(manifest.maxImageSize !== undefined ? { maxImageSize: manifest.maxImageSize } : {}),
|
|
150
|
+
materials: manifest.materials,
|
|
151
|
+
});
|
|
152
|
+
assertExtractRuntimeMatchesManifest(prepared, manifest);
|
|
153
|
+
await replaceStoreWithStaging(storeDir, stagingDir);
|
|
154
|
+
committed = true;
|
|
155
|
+
return { status: await inspectStoreKvCache(storeDir), rebuilt: true };
|
|
156
|
+
}
|
|
157
|
+
finally {
|
|
158
|
+
if (!committed) {
|
|
159
|
+
await rm(stagingDir, { recursive: true, force: true });
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
function automaticRebuildDisabledError(storename, status) {
|
|
164
|
+
const detail = status.hasKvCache && !status.hasIncrementalBase
|
|
165
|
+
? 'the persisted cache does not provide an incremental base'
|
|
166
|
+
: status.issue === 'index-entry-missing'
|
|
167
|
+
? 'the cache index does not reference a usable cache'
|
|
168
|
+
: status.issue === 'index-invalid'
|
|
169
|
+
? 'the cache index is invalid'
|
|
170
|
+
: 'the persisted KV cache is missing or invalid';
|
|
171
|
+
return new Error(`Automatic KV cache rebuild is disabled for store '${storename}': ${detail}. `
|
|
172
|
+
+ 'Run create again or enable autoRebuildCache to add materials.');
|
|
173
|
+
}
|
|
121
174
|
export function assertExtractRuntimeMatchesManifest(runtime, manifest) {
|
|
122
175
|
const expectedProvider = getManifestProvider(manifest);
|
|
123
176
|
if (runtime.provider !== expectedProvider) {
|
|
@@ -162,6 +215,11 @@ export async function appendToExtractStore(options) {
|
|
|
162
215
|
}
|
|
163
216
|
const previousManifest = options.existingManifest
|
|
164
217
|
?? await readExtractStoreManifest(storeDir, options.storename);
|
|
218
|
+
const cacheStatus = await inspectStoreKvCache(storeDir);
|
|
219
|
+
const cacheRebuilt = !cacheStatus.hasIncrementalBase;
|
|
220
|
+
if (cacheRebuilt && options.autoRebuildCache === false) {
|
|
221
|
+
throw automaticRebuildDisabledError(options.storename, cacheStatus);
|
|
222
|
+
}
|
|
165
223
|
const materials = mergeMaterials(previousManifest.materials, options.incomingMaterials);
|
|
166
224
|
const stagingDir = await mkdtemp(join(dirname(storeDir), `.${basename(storeDir)}.add-`));
|
|
167
225
|
let committed = false;
|
|
@@ -202,6 +260,7 @@ export async function appendToExtractStore(options) {
|
|
|
202
260
|
provider: prepared.provider,
|
|
203
261
|
backend: prepared.backend,
|
|
204
262
|
addedMaterials: materials.length - previousManifest.materials.length,
|
|
263
|
+
cacheRebuilt,
|
|
205
264
|
};
|
|
206
265
|
}
|
|
207
266
|
finally {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-store.js","sourceRoot":"","sources":["../src/extract-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"extract-store.js","sourceRoot":"","sources":["../src/extract-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAInE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,YAAY,EACZ,aAAa,GAEd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,WAAW,GAEZ,MAAM,gBAAgB,CAAC;AAyBxB,KAAK,UAAU,2BAA2B,CACxC,OAAuB,EACvB,cAAuB;IAEvB,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,UAAmB,EAAE,CAAC;QAC7B,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;YACjC,MAAM,UAAU,CAAC;QACnB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAmC;IAEnC,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC;QACzC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,GAAG,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtF,CAAC,CAAC;IACH,IAAI,cAAuB,CAAC;IAE5B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,oBAAoB,CAAC;YACnC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,MAAM,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE;YACxC,gBAAgB,EAAE,UAAU;SAC7B,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,OAAO,CAAC;YACpB,GAAG,EAAE,iBAAiB;YACtB,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE;SAC1C,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7C,OAAO;YACL,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,KAAK;YACnC,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,cAAc,GAAG,KAAK,CAAC;QACvB,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,uEAAuE;QACvE,yDAAyD;QACzD,MAAM,2BAA2B,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,QAAuB;IAC1C,OAAO,QAAQ,CAAC,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC;AACvC,CAAC;AAED,SAAS,cAAc,CAAC,IAAmB,EAAE,KAAoB;IAC/D,OAAO,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;WAC5B,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;WAC1B,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;AACtD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC5B,QAAkC,EAClC,QAAkC;IAElC,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACnD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC1C,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QACjC,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAChC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;QACvC,IAAI,QAAQ,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CACb,8CAA8C,EAAE,KAAK;kBACnD,4EAA4E,CAC/E,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB,EAAE,SAAiB;IAC5D,OAAO,IAAI,KAAK,CACd,oBAAoB,QAAQ,IAAI;UAC9B,uCAAuC,SAAS,sBAAsB,CACzE,CAAC;AACJ,CAAC;AAED,uFAAuF;AACvF,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,QAAgB,EAChB,SAAiB;IAEjB,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC;QAC3C,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,CAAC,CAAC,MAAM,cAAc,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,6BAA6B,gBAAgB,EAAE,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,YAAY,CAAC,gBAAgB,CAAC,CAAC;AACxC,CAAC;AAkBD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,OAAkC;IAElC,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC,gBAAgB,KAAK,KAAK,EAAE,CAAC;QAC/D,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACpC,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ;WAC5B,MAAM,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;IAC7F,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjE,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACzE,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa;aAC5B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,kBAAkB,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;aACjF,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAEvF,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC;YACzC,QAAQ,EAAE,UAAU;YACpB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,mBAAmB,CAAC,QAAQ,CAAC;YACvC,GAAG,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,GAAG,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvF,SAAS,EAAE,QAAQ,CAAC,SAAS;SAC9B,CAAC,CAAC;QACH,mCAAmC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxD,MAAM,uBAAuB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACpD,SAAS,GAAG,IAAI,CAAC;QACjB,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACxE,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,EAAE,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;AACH,CAAC;AA2BD,SAAS,6BAA6B,CACpC,SAAiB,EACjB,MAA0B;IAE1B,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,kBAAkB;QAC5D,CAAC,CAAC,0DAA0D;QAC5D,CAAC,CAAC,MAAM,CAAC,KAAK,KAAK,qBAAqB;YACxC,CAAC,CAAC,mDAAmD;YACrD,CAAC,CAAC,MAAM,CAAC,KAAK,KAAK,eAAe;gBAChC,CAAC,CAAC,4BAA4B;gBAC9B,CAAC,CAAC,8CAA8C,CAAC;IACrD,OAAO,IAAI,KAAK,CACd,qDAAqD,SAAS,MAAM,MAAM,IAAI;UAC5E,+DAA+D,CAClE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mCAAmC,CACjD,OAAmD,EACnD,QAA8B;IAE9B,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACvD,IAAI,OAAO,CAAC,QAAQ,KAAK,gBAAgB,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,kDAAkD,gBAAgB,KAAK;cACrE,qBAAqB,OAAO,CAAC,QAAQ,GAAG,CAC3C,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,+CAA+C,QAAQ,CAAC,KAAK,KAAK;cAChE,qBAAqB,OAAO,CAAC,KAAK,GAAG,CACxC,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,uBAAuB,CACpC,QAAgB,EAChB,UAAkB;IAElB,MAAM,SAAS,GAAG,IAAI,CACpB,OAAO,CAAC,QAAQ,CAAC,EACjB,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,UAAU,EAAE,EAAE,CAChD,CAAC;IAEF,MAAM,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,aAAsB,EAAE,CAAC;YAChC,MAAM,IAAI,cAAc,CACtB,CAAC,KAAK,EAAE,aAAa,CAAC,EACtB,yCAAyC,QAAQ,EAAE,CACpD,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,2EAA2E;IAC3E,0DAA0D;IAC1D,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAoC;IAEpC,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,CAAC,CAAC,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QACnC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB;WAC5C,MAAM,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,CAAC,WAAW,CAAC,kBAAkB,CAAC;IACrD,IAAI,YAAY,IAAI,OAAO,CAAC,gBAAgB,KAAK,KAAK,EAAE,CAAC;QACvD,MAAM,6BAA6B,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,cAAc,CAAC,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACxF,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACzF,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAEjE,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC;YACzC,QAAQ,EAAE,UAAU;YACpB,KAAK,EAAE,gBAAgB,CAAC,KAAK;YAC7B,QAAQ,EAAE,mBAAmB,CAAC,gBAAgB,CAAC;YAC/C,GAAG,CAAC,gBAAgB,CAAC,OAAO,KAAK,SAAS;gBACxC,CAAC,CAAC,EAAE,OAAO,EAAE,gBAAgB,CAAC,OAAO,EAAE;gBACvC,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,gBAAgB,CAAC,YAAY,KAAK,SAAS;gBAC7C,CAAC,CAAC,EAAE,YAAY,EAAE,gBAAgB,CAAC,YAAY,EAAE;gBACjD,CAAC,CAAC,EAAE,CAAC;YACP,SAAS;SACV,CAAC,CAAC;QACH,mCAAmC,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QAChE,MAAM,sBAAsB,GAAG,EAAE,GAAG,gBAAgB,EAAE,CAAC;QACvD,OAAO,sBAAsB,CAAC,OAAO,CAAC;QACtC,MAAM,YAAY,GAAyB;YACzC,GAAG,sBAAsB;YACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,SAAS;YACT,GAAG,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,GAAG,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS;gBACrC,CAAC,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE;gBACzC,CAAC,CAAC,EAAE,CAAC;YACP,SAAS,EAAE,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE;SAC/D,CAAC;QACF,MAAM,aAAa,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAE9C,MAAM,uBAAuB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACpD,SAAS,GAAG,IAAI,CAAC;QACjB,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,cAAc,EAAE,SAAS,CAAC,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM;YACpE,YAAY;SACb,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,EAAE,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;AACH,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -36,6 +36,12 @@ export interface ExtractSessionOptions<TContext = ExtractContext> {
|
|
|
36
36
|
corpus: ExtractCorpus;
|
|
37
37
|
/** Output schema (Phase 3: structured output). Accepted at session creation. */
|
|
38
38
|
schema?: object;
|
|
39
|
+
/**
|
|
40
|
+
* Automatically create a missing disk cache. Existing cache hits remain
|
|
41
|
+
* usable when this is false; misses fall back to an uncached query.
|
|
42
|
+
* Defaults to true.
|
|
43
|
+
*/
|
|
44
|
+
autoRebuildCache?: boolean;
|
|
39
45
|
/**
|
|
40
46
|
* Cache preparation policy. Normal sessions are best-effort; store
|
|
41
47
|
* preparation uses `required` so an empty cache handle cannot be committed.
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,QAAQ,EAAE,qBAAqB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACzG,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EACV,WAAW,EACX,cAAc,EACd,aAAa,EACd,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,oEAAoE;AACpE,MAAM,WAAW,aAAa;IAC5B,oCAAoC;IACpC,SAAS,CAAC,EAAE,cAAc,CAAC;IAC3B,iCAAiC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,qBAAqB,CAAC,QAAQ,GAAG,cAAc;IAC9D,MAAM,EAAE,QAAQ,CAAC;IACjB,kDAAkD;IAClD,eAAe,EAAE,qBAAqB,CAAC;IACvC,kEAAkE;IAClE,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,UAAU,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IACpC;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IACtC,kDAAkD;IAClD,MAAM,EAAE,aAAa,CAAC;IACtB,gFAAgF;IAChF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,aAAa,GAAG,UAAU,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,6DAA6D;IAC7D,GAAG,EAAE,MAAM,GAAG,cAAc,CAAC;IAC7B,6CAA6C;IAC7C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,YAAY,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IAC7B,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,0BAA0B;IACzC;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACzD,UAAU,IAAI,aAAa,CAAC,aAAa,CAAC,CAAC;IAC3C,KAAK,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5D"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,QAAQ,EAAE,qBAAqB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACzG,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EACV,WAAW,EACX,cAAc,EACd,aAAa,EACd,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,oEAAoE;AACpE,MAAM,WAAW,aAAa;IAC5B,oCAAoC;IACpC,SAAS,CAAC,EAAE,cAAc,CAAC;IAC3B,iCAAiC;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,qBAAqB,CAAC,QAAQ,GAAG,cAAc;IAC9D,MAAM,EAAE,QAAQ,CAAC;IACjB,kDAAkD;IAClD,eAAe,EAAE,qBAAqB,CAAC;IACvC,kEAAkE;IAClE,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,UAAU,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IACpC;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IACtC,kDAAkD;IAClD,MAAM,EAAE,aAAa,CAAC;IACtB,gFAAgF;IAChF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,aAAa,GAAG,UAAU,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,6DAA6D;IAC7D,GAAG,EAAE,MAAM,GAAG,cAAc,CAAC;IAC7B,6CAA6C;IAC7C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,YAAY,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IAC7B,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,0BAA0B;IACzC;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACzD,UAAU,IAAI,aAAa,CAAC,aAAa,CAAC,CAAC;IAC3C,KAAK,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5D"}
|
package/docs/API.md
CHANGED
|
@@ -133,7 +133,8 @@ const storeDir = resolveStoreDir(resolveDefaultContainerDir(), 'meeting');
|
|
|
133
133
|
|-----|-------------|------------|
|
|
134
134
|
| `mergeMaterials(existing, incoming)` | `MaterialInput[]` → merged `MaterialInput[]` | `id`(省略時は `title`)で順序を保ってマージ。同一 id・同一内容はスキップし、内容差分はエラー |
|
|
135
135
|
| `prepareExtractCache({ cacheDir, model, provider?, backend?, maxImageSize?, materials })` | `Promise<{ model, provider, backend?, maxImageSize? }>`(解決済み model/provider) | prepare cue で corpus を KV prefill し、session/runtime を close。manifest は変更しない |
|
|
136
|
-
| `
|
|
136
|
+
| `ensureStoreKvCache({ storeDir, storename, manifest?, autoRebuildCache? })` | `Promise<{ status, rebuilt }>` | extract 前に cache-index と cache 実体を検査し、必要なら staging store で manifest から full prefill。manifest は変更しない |
|
|
137
|
+
| `appendToExtractStore({ storeDir, storename, incomingMaterials, existingManifest?, now?, autoRebuildCache? })` | `Promise<{ manifest, model, provider, backend?, addedMaterials, cacheRebuilt }>` | 既存 store を staging に複製して incremental prefill と manifest 更新を行い、成功時に rename 交換。incremental base が無い場合は既定で full prefill にフォールバックし、`autoRebuildCache: false` なら明示エラーで停止する。prefill / manifest / runtime の失敗時は元の corpus・manifest・KV を保持 |
|
|
137
138
|
|
|
138
139
|
`appendToExtractStore` は `manifest.provider`(未指定の legacy manifest は MLX)、`manifest.model`、`manifest.backend`(MLX のみ。未指定時は `auto` fallback)、および保存済み `maxImageSize` を使い、provider + model の不一致を検証します。成功時だけ `updatedAt`、materials、解決済み provider/backend、画像 resize 条件を反映します。`readExtractStoreManifest` は store の存在と manifest を検証し、存在しない store には `create` を案内するエラーを返します。ライブラリ層のマージ、prefill、失敗時保全は `src/extract-store.test.ts` で CLI から独立して検証しています。
|
|
139
140
|
|
|
@@ -204,6 +205,7 @@ function createExtractSession<TContext = ExtractContext>(
|
|
|
204
205
|
| `domainModule` | `PromptModule<TContext>` | — | base の上に merge |
|
|
205
206
|
| `corpus` | `ExtractCorpus` | ✅ | セッション固定 corpus |
|
|
206
207
|
| `schema` | `object` | — | JSON Schema(structured output) |
|
|
208
|
+
| `autoRebuildCache` | `boolean` | — | 欠損 cache を自動作成するか(既定 `true`)。`false` では既存 cache の read-only hit のみ利用し、miss は cache を作らず uncached query にフォールバック |
|
|
207
209
|
| `cachePreparation` | `'best-effort' \| 'required'` | — | 通常は `best-effort`(省略時)。`required` は空 handle をエラーにして driver query を実行しない |
|
|
208
210
|
| `maxImageSize` | `number` | — | VLM 画像キャッシュの正規化に使う最大辺。driver の設定と一致させる。runtime 経由では自動設定 |
|
|
209
211
|
|
|
@@ -252,8 +254,8 @@ CLI の `create` / `add` は各入力ファイルを UTF-8 の文字列として
|
|
|
252
254
|
|
|
253
255
|
```bash
|
|
254
256
|
modular-prompt-extract create <storename> [-m <model>] [--provider <mlx|pytorch>] [--dry-run] <files...>
|
|
255
|
-
modular-prompt-extract add <storename> [--dry-run] <files...>
|
|
256
|
-
modular-prompt-extract extract <storename> [--max-tokens <n>] [--dry-run] <query...>
|
|
257
|
+
modular-prompt-extract add <storename> [--auto-rebuild-cache|--no-auto-rebuild-cache] [--dry-run] <files...>
|
|
258
|
+
modular-prompt-extract extract <storename> [--max-tokens <n>] [--auto-rebuild-cache|--no-auto-rebuild-cache] [--dry-run] <query...>
|
|
257
259
|
modular-prompt-extract list
|
|
258
260
|
modular-prompt-extract clean <storename>
|
|
259
261
|
modular-prompt-extract clean --all
|
|
@@ -272,10 +274,12 @@ modular-prompt-extract clean --all -d ~/.modular-prompt/extract-cache
|
|
|
272
274
|
|
|
273
275
|
`<storename>` は create/add/extract/clean の positional 第1引数として必須(`clean --all` を除く)で、`[a-zA-Z0-9][a-zA-Z0-9_-]*` に一致する必要がある。`create`、`add`、`extract`、`list`、`clean` は予約語である。
|
|
274
276
|
|
|
275
|
-
`add <storename> [--dry-run] <files...>` は既存 store の manifest にファイルを追記し、manifest の model で prepare cue を実行する。既存 cache を staging store に複製してから incremental prefill と manifest 更新を行い、成功時にだけ store を入れ替える。prefill または manifest 更新が失敗した場合は元の store を保持する。同じ絶対パス `id` の同一内容はスキップし、内容が異なる場合は `clean` + `create` を案内してエラーにする。
|
|
277
|
+
`add <storename> [--dry-run] <files...>` は既存 store の manifest にファイルを追記し、manifest の model で prepare cue を実行する。既存 cache を staging store に複製してから incremental prefill と manifest 更新を行い、成功時にだけ store を入れ替える。incremental base が無い場合は既定で full prefill にフォールバックし、再生成時だけ stderr に warning を出す。`--no-auto-rebuild-cache` を指定すると、その場合は明示エラーで停止する。prefill または manifest 更新が失敗した場合は元の store を保持する。同じ絶対パス `id` の同一内容はスキップし、内容が異なる場合は `clean` + `create` を案内してエラーにする。
|
|
276
278
|
|
|
277
279
|
`add --dry-run` はマージ後の compile 済みプロンプトを表示し、driver の起動・KV cache の書き込み・manifest の更新を行わない。`add` では `-m`、`--provider`、`--max-tokens` は指定できない。create の `--provider` は `mlx` または `pytorch` を受け付け、以降の add/extract は manifest に保存された provider を使用する。
|
|
278
280
|
|
|
281
|
+
`create` は store と `manifest.json` を作るために必須です。KV cache は manifest から再生成できる派生データであり、`extract` は欠損・破損・`cache-index.json` が指す実体の欠損を検知すると既定で full prefill を行ってから抽出を続けます。`list` の `KV cache: missing` はこの復旧対象を示します。`MODULAR_PROMPT_EXTRACT_AUTO_REBUILD_CACHE=false`(`0` / `no` / `off` も可)または `--no-auto-rebuild-cache` で自動再生成を無効化できます。環境変数が OFF のときに一時的に有効化する場合は `--auto-rebuild-cache` を指定します。CLI 引数が環境変数より優先され、既定値は有効です。ライブラリの `ExtractSessionOptions.autoRebuildCache: false` では既存 cache の read-only hit を利用し、miss は cache を作らず uncached query にフォールバックします。
|
|
282
|
+
|
|
279
283
|
これは破壊的変更であり、旧 CLI 引数形式と旧レイアウト(container 直下の `manifest.json` と cache files)はサポートしない。旧デフォルト `./.extract-cache` の自動検出・自動移行も行わない。既存データを利用する場合は、[README の旧 CLI / キャッシュレイアウトからの手動移行手順](./README.md#旧-cli--キャッシュレイアウトからの移行)に従って、新しいデフォルトまたは `-d` で指定した store container へ移動する。
|
|
280
284
|
|
|
281
285
|
---
|