@modular-prompt/extract 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -23
- package/dist/cache-lifecycle.d.ts +5 -2
- package/dist/cache-lifecycle.d.ts.map +1 -1
- package/dist/cache-lifecycle.js +11 -7
- package/dist/cache-lifecycle.js.map +1 -1
- package/dist/cli/add-command.d.ts +11 -0
- package/dist/cli/add-command.d.ts.map +1 -0
- package/dist/cli/add-command.js +29 -0
- package/dist/cli/add-command.js.map +1 -0
- package/dist/cli/args.d.ts +14 -0
- package/dist/cli/args.d.ts.map +1 -0
- package/dist/cli/args.js +125 -0
- package/dist/cli/args.js.map +1 -0
- package/dist/cli/clean-command.d.ts +10 -0
- package/dist/cli/clean-command.d.ts.map +1 -0
- package/dist/cli/clean-command.js +26 -0
- package/dist/cli/clean-command.js.map +1 -0
- package/dist/cli/constants.d.ts +4 -4
- package/dist/cli/constants.d.ts.map +1 -1
- package/dist/cli/constants.js +8 -4
- package/dist/cli/constants.js.map +1 -1
- package/dist/cli/create-command.d.ts +3 -1
- package/dist/cli/create-command.d.ts.map +1 -1
- package/dist/cli/create-command.js +29 -28
- package/dist/cli/create-command.js.map +1 -1
- package/dist/cli/extract-command.d.ts +2 -0
- package/dist/cli/extract-command.d.ts.map +1 -1
- package/dist/cli/extract-command.js +5 -3
- package/dist/cli/extract-command.js.map +1 -1
- package/dist/cli/list-command.d.ts +15 -0
- package/dist/cli/list-command.d.ts.map +1 -0
- package/dist/cli/list-command.js +69 -0
- package/dist/cli/list-command.js.map +1 -0
- package/dist/cli/manifest.d.ts +3 -0
- package/dist/cli/manifest.d.ts.map +1 -1
- package/dist/cli/manifest.js +4 -1
- package/dist/cli/manifest.js.map +1 -1
- package/dist/cli/store.d.ts +9 -0
- package/dist/cli/store.d.ts.map +1 -0
- package/dist/cli/store.js +36 -0
- package/dist/cli/store.js.map +1 -0
- package/dist/cli.js +62 -73
- package/dist/cli.js.map +1 -1
- package/dist/create-extract-session.d.ts.map +1 -1
- package/dist/create-extract-session.js +3 -3
- package/dist/create-extract-session.js.map +1 -1
- package/dist/create-mlx-extract-runtime.d.ts +2 -1
- package/dist/create-mlx-extract-runtime.d.ts.map +1 -1
- package/dist/create-mlx-extract-runtime.js +47 -17
- package/dist/create-mlx-extract-runtime.js.map +1 -1
- package/dist/default-models.d.ts +14 -0
- package/dist/default-models.d.ts.map +1 -0
- package/dist/default-models.js +20 -0
- package/dist/default-models.js.map +1 -0
- package/dist/extract-store.d.ts +53 -0
- package/dist/extract-store.d.ts.map +1 -0
- package/dist/extract-store.js +174 -0
- package/dist/extract-store.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/model-resolution.d.ts +25 -0
- package/dist/model-resolution.d.ts.map +1 -0
- package/dist/model-resolution.js +57 -0
- package/dist/model-resolution.js.map +1 -0
- package/dist/types.d.ts +5 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
|
@@ -1,48 +1,49 @@
|
|
|
1
|
-
import { mkdir } from 'node:fs/promises';
|
|
1
|
+
import { mkdir, rm } from 'node:fs/promises';
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { CACHE_PREPARE_CUE, DEFAULT_MODEL } from './constants.js';
|
|
3
|
+
import { prepareExtractCache } from '../extract-store.js';
|
|
4
|
+
import { CACHE_PREPARE_CUE } from './constants.js';
|
|
6
5
|
import { loadMaterialsFromFiles } from './load-materials.js';
|
|
7
|
-
import {
|
|
6
|
+
import { writeManifest } from './manifest.js';
|
|
8
7
|
import { renderExtractPrompt } from './render-prompt.js';
|
|
8
|
+
import { resolveStoreDir, storeExists } from './store.js';
|
|
9
9
|
export async function runCreateCommand(options) {
|
|
10
|
-
const
|
|
11
|
-
const
|
|
10
|
+
const containerDir = resolve(options.cacheDir);
|
|
11
|
+
const storeDir = resolveStoreDir(containerDir, options.storename);
|
|
12
12
|
const materials = await loadMaterialsFromFiles(options.files);
|
|
13
13
|
const request = { cue: CACHE_PREPARE_CUE };
|
|
14
14
|
if (options.dryRun) {
|
|
15
15
|
return renderExtractPrompt({ materials }, request);
|
|
16
16
|
}
|
|
17
|
-
if (await
|
|
18
|
-
throw new Error(`
|
|
19
|
-
+
|
|
17
|
+
if (await storeExists(storeDir)) {
|
|
18
|
+
throw new Error(`Store already exists: ${storeDir}\n`
|
|
19
|
+
+ `Run \`modular-extract clean ${options.storename}\` before creating it again.`);
|
|
20
20
|
}
|
|
21
|
-
await mkdir(
|
|
22
|
-
|
|
21
|
+
await mkdir(storeDir, { recursive: true });
|
|
22
|
+
let storeReady = false;
|
|
23
|
+
let preparedModel;
|
|
23
24
|
try {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
corpus: { materials },
|
|
29
|
-
});
|
|
30
|
-
await session.extract({
|
|
31
|
-
cue: CACHE_PREPARE_CUE,
|
|
32
|
-
options: { maxTokens: 1, temperature: 0 },
|
|
25
|
+
preparedModel = await prepareExtractCache({
|
|
26
|
+
cacheDir: storeDir,
|
|
27
|
+
model: options.model,
|
|
28
|
+
materials,
|
|
33
29
|
});
|
|
34
|
-
|
|
35
|
-
await writeManifest(
|
|
30
|
+
const createdAt = new Date().toISOString();
|
|
31
|
+
await writeManifest(storeDir, {
|
|
36
32
|
version: 1,
|
|
37
|
-
|
|
33
|
+
storename: options.storename,
|
|
34
|
+
model: preparedModel,
|
|
38
35
|
materials,
|
|
39
|
-
createdAt
|
|
36
|
+
createdAt,
|
|
37
|
+
updatedAt: createdAt,
|
|
40
38
|
});
|
|
39
|
+
storeReady = true;
|
|
41
40
|
}
|
|
42
41
|
finally {
|
|
43
|
-
|
|
42
|
+
if (!storeReady) {
|
|
43
|
+
await rm(storeDir, { recursive: true, force: true });
|
|
44
|
+
}
|
|
44
45
|
}
|
|
45
|
-
console.error(`Cache prepared: ${
|
|
46
|
-
console.error(`Materials: ${materials.length} file(s), model: ${
|
|
46
|
+
console.error(`Cache prepared: ${storeDir}`);
|
|
47
|
+
console.error(`Materials: ${materials.length} file(s), model: ${preparedModel}`);
|
|
47
48
|
}
|
|
48
49
|
//# sourceMappingURL=create-command.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-command.js","sourceRoot":"","sources":["../../src/cli/create-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"create-command.js","sourceRoot":"","sources":["../../src/cli/create-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAW1D,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAA6B;IAClE,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC;IAE3C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,mBAAmB,CAAC,EAAE,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,MAAM,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,yBAAyB,QAAQ,IAAI;cACnC,+BAA+B,OAAO,CAAC,SAAS,8BAA8B,CACjF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,aAAiC,CAAC;IACtC,IAAI,CAAC;QACH,aAAa,GAAG,MAAM,mBAAmB,CAAC;YACxC,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,SAAS;SACV,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE3C,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,KAAK,EAAE,aAAa;YACpB,SAAS;YACT,SAAS;YACT,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;QACH,UAAU,GAAG,IAAI,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,KAAK,CAAC,cAAc,SAAS,CAAC,MAAM,oBAAoB,aAAa,EAAE,CAAC,CAAC;AACnF,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-command.d.ts","sourceRoot":"","sources":["../../src/cli/extract-command.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extract-command.d.ts","sourceRoot":"","sources":["../../src/cli/extract-command.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,qBAAqB;IACpC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyCvF"}
|
|
@@ -4,12 +4,14 @@ import { createMlxExtractRuntime } from '../create-mlx-extract-runtime.js';
|
|
|
4
4
|
import { DEFAULT_MAX_TOKENS } from './constants.js';
|
|
5
5
|
import { readManifest } from './manifest.js';
|
|
6
6
|
import { renderExtractPrompt } from './render-prompt.js';
|
|
7
|
+
import { resolveStoreDir } from './store.js';
|
|
7
8
|
export async function runExtractCommand(options) {
|
|
8
|
-
const
|
|
9
|
-
const
|
|
9
|
+
const containerDir = resolve(options.cacheDir);
|
|
10
|
+
const storeDir = resolveStoreDir(containerDir, options.storename);
|
|
10
11
|
if (!options.query.trim()) {
|
|
11
12
|
throw new Error('Query text is required');
|
|
12
13
|
}
|
|
14
|
+
const manifest = await readManifest(storeDir);
|
|
13
15
|
const request = {
|
|
14
16
|
cue: options.query,
|
|
15
17
|
options: {
|
|
@@ -22,7 +24,7 @@ export async function runExtractCommand(options) {
|
|
|
22
24
|
}
|
|
23
25
|
const runtime = await createMlxExtractRuntime({
|
|
24
26
|
model: manifest.model,
|
|
25
|
-
cacheDir,
|
|
27
|
+
cacheDir: storeDir,
|
|
26
28
|
});
|
|
27
29
|
try {
|
|
28
30
|
const session = createExtractSession({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-command.js","sourceRoot":"","sources":["../../src/cli/extract-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"extract-command.js","sourceRoot":"","sources":["../../src/cli/extract-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAW7C,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAA8B;IACpE,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAElE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IAE9C,MAAM,OAAO,GAAG;QACd,GAAG,EAAE,OAAO,CAAC,KAAK;QAClB,OAAO,EAAE;YACP,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;YAClD,WAAW,EAAE,CAAU;SACxB;KACF,CAAC;IAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,mBAAmB,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC;QAC5C,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,QAAQ,EAAE,QAAQ;KACnB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,oBAAoB,CAAC;YACnC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE;SAC1C,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7C,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export interface ListCommandOptions {
|
|
2
|
+
/** Container directory containing one subdirectory per store. */
|
|
3
|
+
cacheDir: string;
|
|
4
|
+
}
|
|
5
|
+
export interface StoreSummary {
|
|
6
|
+
storename: string;
|
|
7
|
+
model: string;
|
|
8
|
+
materialTitles: string[];
|
|
9
|
+
createdAt: string;
|
|
10
|
+
updatedAt?: string;
|
|
11
|
+
hasKvCache: boolean;
|
|
12
|
+
}
|
|
13
|
+
export declare function listStores(cacheDir: string): Promise<StoreSummary[]>;
|
|
14
|
+
export declare function runListCommand(options: ListCommandOptions): Promise<string>;
|
|
15
|
+
//# sourceMappingURL=list-command.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-command.d.ts","sourceRoot":"","sources":["../../src/cli/list-command.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;CACrB;AAsBD,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAuB1E;AAmBD,wBAAsB,cAAc,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAOjF"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { readdir } from 'node:fs/promises';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
import { isKvCacheFile } from './store.js';
|
|
4
|
+
import { manifestExists, readManifest } from './manifest.js';
|
|
5
|
+
async function listStoreDirectories(containerDir) {
|
|
6
|
+
try {
|
|
7
|
+
const entries = await readdir(containerDir, { withFileTypes: true });
|
|
8
|
+
return entries
|
|
9
|
+
.filter((entry) => entry.isDirectory() && !entry.name.startsWith('.'))
|
|
10
|
+
.map((entry) => entry.name)
|
|
11
|
+
.sort((left, right) => left.localeCompare(right));
|
|
12
|
+
}
|
|
13
|
+
catch (error) {
|
|
14
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
|
|
15
|
+
return [];
|
|
16
|
+
}
|
|
17
|
+
throw error;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
async function hasKvCache(storeDir) {
|
|
21
|
+
const entries = await readdir(storeDir, { withFileTypes: true });
|
|
22
|
+
return entries.some((entry) => entry.isFile() && isKvCacheFile(entry.name));
|
|
23
|
+
}
|
|
24
|
+
export async function listStores(cacheDir) {
|
|
25
|
+
const containerDir = resolve(cacheDir);
|
|
26
|
+
const storeNames = await listStoreDirectories(containerDir);
|
|
27
|
+
const summaries = [];
|
|
28
|
+
for (const storename of storeNames) {
|
|
29
|
+
const storeDir = resolve(containerDir, storename);
|
|
30
|
+
if (!(await manifestExists(storeDir))) {
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
const manifest = await readManifest(storeDir);
|
|
34
|
+
summaries.push({
|
|
35
|
+
storename: manifest.storename ?? storename,
|
|
36
|
+
model: manifest.model,
|
|
37
|
+
materialTitles: manifest.materials.map((material) => material.title),
|
|
38
|
+
createdAt: manifest.createdAt,
|
|
39
|
+
updatedAt: manifest.updatedAt,
|
|
40
|
+
hasKvCache: await hasKvCache(storeDir),
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
return summaries;
|
|
44
|
+
}
|
|
45
|
+
function formatSummary(summary) {
|
|
46
|
+
const materials = summary.materialTitles.length > 0
|
|
47
|
+
? ` (${summary.materialTitles.join(', ')})`
|
|
48
|
+
: '';
|
|
49
|
+
const lines = [
|
|
50
|
+
`Store: ${summary.storename}`,
|
|
51
|
+
` Model: ${summary.model}`,
|
|
52
|
+
` Materials: ${summary.materialTitles.length}${materials}`,
|
|
53
|
+
` Created: ${summary.createdAt}`,
|
|
54
|
+
];
|
|
55
|
+
if (summary.updatedAt) {
|
|
56
|
+
lines.push(` Updated: ${summary.updatedAt}`);
|
|
57
|
+
}
|
|
58
|
+
lines.push(` KV cache: ${summary.hasKvCache ? 'present' : 'missing'}`);
|
|
59
|
+
return lines.join('\n');
|
|
60
|
+
}
|
|
61
|
+
export async function runListCommand(options) {
|
|
62
|
+
const containerDir = resolve(options.cacheDir);
|
|
63
|
+
const summaries = await listStores(containerDir);
|
|
64
|
+
if (summaries.length === 0) {
|
|
65
|
+
return `No stores found in: ${containerDir}`;
|
|
66
|
+
}
|
|
67
|
+
return summaries.map(formatSummary).join('\n\n');
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=list-command.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-command.js","sourceRoot":"","sources":["../../src/cli/list-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAgB7D,KAAK,UAAU,oBAAoB,CAAC,YAAoB;IACtD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACrE,OAAO,OAAO;aACX,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACrE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;aAC1B,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACjE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB;IAC/C,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,MAAM,oBAAoB,CAAC,YAAY,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAmB,EAAE,CAAC;IAErC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAClD,IAAI,CAAC,CAAC,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACtC,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC9C,SAAS,CAAC,IAAI,CAAC;YACb,SAAS,EAAE,QAAQ,CAAC,SAAS,IAAI,SAAS;YAC1C,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,cAAc,EAAE,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;YACpE,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,UAAU,EAAE,MAAM,UAAU,CAAC,QAAQ,CAAC;SACvC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,OAAqB;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;QACjD,CAAC,CAAC,KAAK,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QAC3C,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,KAAK,GAAG;QACZ,UAAU,OAAO,CAAC,SAAS,EAAE;QAC7B,YAAY,OAAO,CAAC,KAAK,EAAE;QAC3B,gBAAgB,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,SAAS,EAAE;QAC3D,cAAc,OAAO,CAAC,SAAS,EAAE;KAClC,CAAC;IACF,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAChD,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IACxE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA2B;IAC9D,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC;IACjD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,uBAAuB,YAAY,EAAE,CAAC;IAC/C,CAAC;IACD,OAAO,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACnD,CAAC"}
|
package/dist/cli/manifest.d.ts
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import type { MaterialInput } from '../extract-elements.js';
|
|
2
2
|
export interface ExtractCacheManifest {
|
|
3
3
|
version: 1;
|
|
4
|
+
/** Store name for self-describing cache directories. */
|
|
5
|
+
storename?: string;
|
|
4
6
|
model: string;
|
|
5
7
|
materials: MaterialInput[];
|
|
6
8
|
createdAt: string;
|
|
9
|
+
updatedAt?: string;
|
|
7
10
|
}
|
|
8
11
|
export declare function manifestPath(cacheDir: string): string;
|
|
9
12
|
export declare function manifestExists(cacheDir: string): Promise<boolean>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/cli/manifest.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAG5D,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,CAAC,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/cli/manifest.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAG5D,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,CAAC,CAAC;IACX,wDAAwD;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAOvE;AAED,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAYlF;AAED,wBAAsB,aAAa,CACjC,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAMf"}
|
package/dist/cli/manifest.js
CHANGED
|
@@ -16,7 +16,10 @@ export async function manifestExists(cacheDir) {
|
|
|
16
16
|
export async function readManifest(cacheDir) {
|
|
17
17
|
const raw = await readFile(manifestPath(cacheDir), 'utf-8');
|
|
18
18
|
const parsed = JSON.parse(raw);
|
|
19
|
-
if (parsed.version !== 1
|
|
19
|
+
if (parsed.version !== 1
|
|
20
|
+
|| !parsed.model
|
|
21
|
+
|| !Array.isArray(parsed.materials)
|
|
22
|
+
|| (parsed.updatedAt !== undefined && typeof parsed.updatedAt !== 'string')) {
|
|
20
23
|
throw new Error(`Invalid manifest: ${manifestPath(cacheDir)}`);
|
|
21
24
|
}
|
|
22
25
|
return parsed;
|
package/dist/cli/manifest.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/cli/manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/cli/manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAYnD,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IACnD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;QACrC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAyB,CAAC;IACvD,IACE,MAAM,CAAC,OAAO,KAAK,CAAC;WACjB,CAAC,MAAM,CAAC,KAAK;WACb,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;WAChC,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,EAC3E,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB,EAChB,QAA8B;IAE9B,MAAM,SAAS,CACb,YAAY,CAAC,QAAQ,CAAC,EACtB,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EACxC,OAAO,CACR,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Validate a store name used as a single directory component. */
|
|
2
|
+
export declare function validateStorename(storename: string): void;
|
|
3
|
+
/** Resolve a validated store directory below the container directory. */
|
|
4
|
+
export declare function resolveStoreDir(containerDir: string, storename: string): string;
|
|
5
|
+
/** Return whether a path already exists, regardless of its file type. */
|
|
6
|
+
export declare function storeExists(storeDir: string): Promise<boolean>;
|
|
7
|
+
/** Return whether the store contains at least one persisted KV cache archive. */
|
|
8
|
+
export declare function isKvCacheFile(filename: string): boolean;
|
|
9
|
+
//# sourceMappingURL=store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/cli/store.ts"],"names":[],"mappings":"AAMA,kEAAkE;AAClE,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAUzD;AAED,yEAAyE;AACzE,wBAAgB,eAAe,CAAC,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAG/E;AAED,yEAAyE;AACzE,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAUpE;AAED,iFAAiF;AACjF,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAEvD"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { stat } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
const STORE_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/;
|
|
4
|
+
const RESERVED_STORE_NAMES = new Set(['create', 'add', 'extract', 'list', 'clean']);
|
|
5
|
+
/** Validate a store name used as a single directory component. */
|
|
6
|
+
export function validateStorename(storename) {
|
|
7
|
+
if (typeof storename !== 'string' || !STORE_NAME_PATTERN.test(storename)) {
|
|
8
|
+
throw new Error(`Invalid storename '${String(storename)}': must match [a-zA-Z0-9][a-zA-Z0-9_-]*`);
|
|
9
|
+
}
|
|
10
|
+
if (RESERVED_STORE_NAMES.has(storename)) {
|
|
11
|
+
throw new Error(`Invalid storename '${storename}': reserved name`);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
/** Resolve a validated store directory below the container directory. */
|
|
15
|
+
export function resolveStoreDir(containerDir, storename) {
|
|
16
|
+
validateStorename(storename);
|
|
17
|
+
return join(containerDir, storename);
|
|
18
|
+
}
|
|
19
|
+
/** Return whether a path already exists, regardless of its file type. */
|
|
20
|
+
export async function storeExists(storeDir) {
|
|
21
|
+
try {
|
|
22
|
+
await stat(storeDir);
|
|
23
|
+
return true;
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
throw error;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/** Return whether the store contains at least one persisted KV cache archive. */
|
|
33
|
+
export function isKvCacheFile(filename) {
|
|
34
|
+
return filename.endsWith('.safetensors.zip');
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/cli/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,kBAAkB,GAAG,6BAA6B,CAAC;AACzD,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAEpF,kEAAkE;AAClE,MAAM,UAAU,iBAAiB,CAAC,SAAiB;IACjD,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CACb,sBAAsB,MAAM,CAAC,SAAS,CAAC,yCAAyC,CACjF,CAAC;IACJ,CAAC;IAED,IAAI,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,sBAAsB,SAAS,kBAAkB,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAAC,YAAoB,EAAE,SAAiB;IACrE,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC7B,OAAO,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;AACvC,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAgB;IAChD,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,OAAO,QAAQ,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC;AAC/C,CAAC"}
|
package/dist/cli.js
CHANGED
|
@@ -2,91 +2,52 @@
|
|
|
2
2
|
import { readFileSync } from 'node:fs';
|
|
3
3
|
import { dirname, join } from 'node:path';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
5
|
-
import {
|
|
5
|
+
import { DEFAULT_MAX_TOKENS, resolveDefaultContainerDir } from './cli/constants.js';
|
|
6
|
+
import { parseArgs } from './cli/args.js';
|
|
7
|
+
import { runAddCommand } from './cli/add-command.js';
|
|
8
|
+
import { runCleanCommand } from './cli/clean-command.js';
|
|
6
9
|
import { runCreateCommand } from './cli/create-command.js';
|
|
7
10
|
import { runExtractCommand } from './cli/extract-command.js';
|
|
11
|
+
import { runListCommand } from './cli/list-command.js';
|
|
8
12
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
9
13
|
const packageJson = JSON.parse(readFileSync(join(__dirname, '../package.json'), 'utf-8'));
|
|
10
14
|
function printHelp() {
|
|
11
15
|
console.log(`modular-extract v${packageJson.version}
|
|
12
16
|
|
|
13
17
|
Usage:
|
|
14
|
-
modular-extract create [-d <cache-dir>] [-m <model>] [--dry-run] <files...>
|
|
15
|
-
modular-extract
|
|
18
|
+
modular-extract create <storename> [-d <cache-dir>] [-m <model>] [--dry-run] <files...>
|
|
19
|
+
modular-extract add <storename> [-d <cache-dir>] [--dry-run] <files...>
|
|
20
|
+
modular-extract extract <storename> [-d <cache-dir>] [--max-tokens <n>] [--dry-run] <query...>
|
|
21
|
+
modular-extract list [-d <cache-dir>]
|
|
22
|
+
modular-extract clean <storename> [-d <cache-dir>]
|
|
23
|
+
modular-extract clean --all [-d <cache-dir>]
|
|
16
24
|
|
|
17
25
|
Commands:
|
|
18
|
-
create Load input files and prepare KV cache in <cache-dir>
|
|
19
|
-
|
|
26
|
+
create Load input files and prepare KV cache in <cache-dir>/<storename>
|
|
27
|
+
add Append input files and incrementally extend an existing store
|
|
28
|
+
extract Run extraction query against a prepared store
|
|
29
|
+
list List stores and their cache summaries
|
|
30
|
+
clean Remove one store or the entire cache container
|
|
20
31
|
|
|
21
32
|
Options:
|
|
22
|
-
-d, --cache-dir <path>
|
|
23
|
-
-m, --model <model> MLX model
|
|
33
|
+
-d, --cache-dir <path> Store container directory (default: ${resolveDefaultContainerDir()})
|
|
34
|
+
-m, --model <model> MLX model alias from models.yaml or raw model id
|
|
24
35
|
--max-tokens <n> Max tokens for extract (default: ${DEFAULT_MAX_TOKENS})
|
|
25
36
|
--dry-run Compile and print full prompt text (no MLX / no cache write)
|
|
37
|
+
--all Remove the entire cache container (clean only)
|
|
26
38
|
-h, --help Show help
|
|
27
39
|
|
|
28
|
-
|
|
29
|
-
|
|
40
|
+
Store name:
|
|
41
|
+
Must match [a-zA-Z0-9][a-zA-Z0-9_-]* and cannot be create, add, extract, list, or clean.
|
|
30
42
|
|
|
31
43
|
Note:
|
|
44
|
+
Without -m, models.default (or the first model entry) is selected from bundled config merged with
|
|
45
|
+
~/.modular-prompt/models.yaml (MODULAR_PROMPT_HOME can override its location).
|
|
46
|
+
MLX_MODEL is also supported for backward compatibility as the bundled default; user yaml overrides it.
|
|
47
|
+
If no model is configured, specify -m <model-id-or-alias> or define models.default.
|
|
32
48
|
MLX backend is fixed to mlx-lm (backend: lm) for prompt cache support.
|
|
33
49
|
`);
|
|
34
50
|
}
|
|
35
|
-
function parseArgs(argv) {
|
|
36
|
-
const result = { positional: [] };
|
|
37
|
-
let index = 0;
|
|
38
|
-
while (index < argv.length) {
|
|
39
|
-
const arg = argv[index];
|
|
40
|
-
if (arg === '-h' || arg === '--help') {
|
|
41
|
-
result.command = 'help';
|
|
42
|
-
return result;
|
|
43
|
-
}
|
|
44
|
-
if (!result.command && !arg.startsWith('-')) {
|
|
45
|
-
if (arg === 'create' || arg === 'extract') {
|
|
46
|
-
result.command = arg;
|
|
47
|
-
index += 1;
|
|
48
|
-
continue;
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
if (arg === '-d' || arg === '--cache-dir') {
|
|
52
|
-
result.cacheDir = argv[index + 1];
|
|
53
|
-
if (!result.cacheDir) {
|
|
54
|
-
throw new Error(`${arg} requires a path`);
|
|
55
|
-
}
|
|
56
|
-
index += 2;
|
|
57
|
-
continue;
|
|
58
|
-
}
|
|
59
|
-
if (arg === '-m' || arg === '--model') {
|
|
60
|
-
result.model = argv[index + 1];
|
|
61
|
-
if (!result.model) {
|
|
62
|
-
throw new Error(`${arg} requires a model id`);
|
|
63
|
-
}
|
|
64
|
-
index += 2;
|
|
65
|
-
continue;
|
|
66
|
-
}
|
|
67
|
-
if (arg === '--dry-run') {
|
|
68
|
-
result.dryRun = true;
|
|
69
|
-
index += 1;
|
|
70
|
-
continue;
|
|
71
|
-
}
|
|
72
|
-
if (arg === '--max-tokens') {
|
|
73
|
-
const value = argv[index + 1];
|
|
74
|
-
const parsed = Number.parseInt(value ?? '', 10);
|
|
75
|
-
if (!Number.isFinite(parsed) || parsed < 1) {
|
|
76
|
-
throw new Error('--max-tokens must be a positive integer');
|
|
77
|
-
}
|
|
78
|
-
result.maxTokens = parsed;
|
|
79
|
-
index += 2;
|
|
80
|
-
continue;
|
|
81
|
-
}
|
|
82
|
-
if (arg.startsWith('-')) {
|
|
83
|
-
throw new Error(`Unknown option: ${arg}`);
|
|
84
|
-
}
|
|
85
|
-
result.positional.push(arg);
|
|
86
|
-
index += 1;
|
|
87
|
-
}
|
|
88
|
-
return result;
|
|
89
|
-
}
|
|
90
51
|
async function main() {
|
|
91
52
|
const parsed = parseArgs(process.argv.slice(2));
|
|
92
53
|
if (!parsed.command || parsed.command === 'help') {
|
|
@@ -96,10 +57,12 @@ async function main() {
|
|
|
96
57
|
}
|
|
97
58
|
return;
|
|
98
59
|
}
|
|
60
|
+
const cacheDir = parsed.cacheDir ?? resolveDefaultContainerDir();
|
|
99
61
|
if (parsed.command === 'create') {
|
|
100
62
|
const output = await runCreateCommand({
|
|
101
|
-
cacheDir
|
|
102
|
-
|
|
63
|
+
cacheDir,
|
|
64
|
+
storename: parsed.storename,
|
|
65
|
+
model: parsed.model,
|
|
103
66
|
files: parsed.positional,
|
|
104
67
|
dryRun: parsed.dryRun,
|
|
105
68
|
});
|
|
@@ -108,14 +71,40 @@ async function main() {
|
|
|
108
71
|
}
|
|
109
72
|
return;
|
|
110
73
|
}
|
|
111
|
-
if (
|
|
112
|
-
|
|
74
|
+
if (parsed.command === 'add') {
|
|
75
|
+
const output = await runAddCommand({
|
|
76
|
+
cacheDir,
|
|
77
|
+
storename: parsed.storename,
|
|
78
|
+
files: parsed.positional,
|
|
79
|
+
dryRun: parsed.dryRun,
|
|
80
|
+
});
|
|
81
|
+
if (typeof output === 'string') {
|
|
82
|
+
process.stdout.write(`${output}\n`);
|
|
83
|
+
}
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
if (parsed.command === 'extract') {
|
|
87
|
+
const text = await runExtractCommand({
|
|
88
|
+
cacheDir,
|
|
89
|
+
storename: parsed.storename,
|
|
90
|
+
query: parsed.positional.join(' '),
|
|
91
|
+
maxTokens: parsed.maxTokens,
|
|
92
|
+
dryRun: parsed.dryRun,
|
|
93
|
+
});
|
|
94
|
+
process.stdout.write(`${text}\n`);
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if (parsed.command === 'clean') {
|
|
98
|
+
const message = await runCleanCommand({
|
|
99
|
+
cacheDir,
|
|
100
|
+
storename: parsed.storename,
|
|
101
|
+
all: parsed.all,
|
|
102
|
+
});
|
|
103
|
+
process.stdout.write(`${message}\n`);
|
|
104
|
+
return;
|
|
113
105
|
}
|
|
114
|
-
const text = await
|
|
115
|
-
cacheDir
|
|
116
|
-
query: parsed.positional.join(' '),
|
|
117
|
-
maxTokens: parsed.maxTokens,
|
|
118
|
-
dryRun: parsed.dryRun,
|
|
106
|
+
const text = await runListCommand({
|
|
107
|
+
cacheDir,
|
|
119
108
|
});
|
|
120
109
|
process.stdout.write(`${text}\n`);
|
|
121
110
|
}
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AACpF,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAC5B,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,OAAO,CAAC,CACnC,CAAC;AAEzB,SAAS,SAAS;IAChB,OAAO,CAAC,GAAG,CAAC,oBAAoB,WAAW,CAAC,OAAO;;;;;;;;;;;;;;;;;;iEAkBY,0BAA0B,EAAE;;8DAE/B,kBAAkB;;;;;;;;;;;;;;CAc/E,CAAC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhD,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QACjD,SAAS,EAAE,CAAC;QACZ,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,0BAA0B,EAAE,CAAC;IAEjE,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC;YACpC,QAAQ;YACR,SAAS,EAAE,MAAM,CAAC,SAAU;YAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,KAAK,EAAE,MAAM,CAAC,UAAU;YACxB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QACH,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC;QACtC,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YACjC,QAAQ;YACR,SAAS,EAAE,MAAM,CAAC,SAAU;YAC5B,KAAK,EAAE,MAAM,CAAC,UAAU;YACxB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QACH,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC;QACtC,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC;YACnC,QAAQ;YACR,SAAS,EAAE,MAAM,CAAC,SAAU;YAC5B,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;YAClC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC;YACpC,QAAQ;YACR,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QACrC,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC;QAChC,QAAQ;KACT,CAAC,CAAC;IACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC;AACpC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC9B,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-extract-session.d.ts","sourceRoot":"","sources":["../src/create-extract-session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAiC,cAAc,EAA8B,qBAAqB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"create-extract-session.d.ts","sourceRoot":"","sources":["../src/create-extract-session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAiC,cAAc,EAA8B,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAyBnI,wBAAgB,oBAAoB,CAAC,QAAQ,GAAG,cAAc,EAC5D,OAAO,EAAE,qBAAqB,CAAC,QAAQ,CAAC,GACvC,cAAc,CA2EhB"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { merge } from '@modular-prompt/core';
|
|
2
2
|
import { resolveSessionModules } from './resolve-session-modules.js';
|
|
3
3
|
import { compileExtractPrompt } from './compile-extract-prompt.js';
|
|
4
|
-
import { ensureCacheControllerReady, prepareSessionCache, releaseSessionCache,
|
|
4
|
+
import { ensureCacheControllerReady, prepareSessionCache, releaseSessionCache, } from './cache-lifecycle.js';
|
|
5
5
|
function buildSessionBaseModule(baseModule, schema) {
|
|
6
6
|
if (!schema) {
|
|
7
7
|
return baseModule;
|
|
@@ -13,7 +13,7 @@ function buildSessionBaseModule(baseModule, schema) {
|
|
|
13
13
|
}
|
|
14
14
|
export function createExtractSession(options) {
|
|
15
15
|
const { driver, corpus, schema, cacheController, baseModule } = options;
|
|
16
|
-
const model =
|
|
16
|
+
const model = options.model;
|
|
17
17
|
const sessionBaseModule = buildSessionBaseModule(resolveSessionModules(options), schema);
|
|
18
18
|
const history = [];
|
|
19
19
|
const cacheState = {
|
|
@@ -27,7 +27,7 @@ export function createExtractSession(options) {
|
|
|
27
27
|
throw new Error('ExtractSession is closed');
|
|
28
28
|
}
|
|
29
29
|
await ensureCacheControllerReady(driver, cacheState);
|
|
30
|
-
await prepareSessionCache(cacheController, model, sessionBaseModule, corpus, request, baseModule, cacheState);
|
|
30
|
+
await prepareSessionCache(cacheController, model, sessionBaseModule, corpus, request, baseModule, cacheState, { required: options.cachePreparation === 'required' });
|
|
31
31
|
const compiled = compileExtractPrompt(sessionBaseModule, corpus, request, baseModule);
|
|
32
32
|
const queryOptions = cacheState.handle
|
|
33
33
|
? {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-extract-session.js","sourceRoot":"","sources":["../src/create-extract-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAI7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EACL,0BAA0B,EAC1B,mBAAmB,EACnB,mBAAmB,
|
|
1
|
+
{"version":3,"file":"create-extract-session.js","sourceRoot":"","sources":["../src/create-extract-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAI7C,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EACL,0BAA0B,EAC1B,mBAAmB,EACnB,mBAAmB,GAEpB,MAAM,sBAAsB,CAAC;AAE9B,SAAS,sBAAsB,CAC7B,UAAkC,EAClC,MAAe;IAEf,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,MAAM,YAAY,GAAiB;QACjC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;KAC5C,CAAC;IAEF,OAAO,KAAK,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,OAAwC;IAExC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAE5B,MAAM,iBAAiB,GAAG,sBAAsB,CAC9C,qBAAqB,CAAC,OAAO,CAAC,EAC9B,MAAM,CACP,CAAC;IACF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,UAAU,GAAwB;QACtC,MAAM,EAAE,IAAI;QACZ,eAAe,EAAE,KAAK;KACvB,CAAC;IACF,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,OAAO;QACL,KAAK,CAAC,OAAO,CAAC,OAAuB;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAC9C,CAAC;YAED,MAAM,0BAA0B,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YACrD,MAAM,mBAAmB,CACvB,eAAe,EACf,KAAK,EACL,iBAAiB,EACjB,MAAM,EACN,OAAO,EACP,UAAU,EACV,UAAU,EACV,EAAE,QAAQ,EAAE,OAAO,CAAC,gBAAgB,KAAK,UAAU,EAAE,CACtD,CAAC;YAEF,MAAM,QAAQ,GAAG,oBAAoB,CACnC,iBAAiB,EACjB,MAAM,EACN,OAAO,EACP,UAAU,CACX,CAAC;YACF,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM;gBACpC,CAAC,CAAC;oBACE,GAAG,OAAO,CAAC,OAAO;oBAClB,KAAK,EAAE,KAAc;oBACrB,WAAW,EAAE,UAAU,CAAC,MAAM;iBAC/B;gBACH,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;YAEpB,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YAE/D,MAAM,MAAM,GAAkB;gBAC5B,IAAI,EAAE,WAAW,CAAC,OAAO;gBACzB,UAAU,EAAE,WAAW,CAAC,gBAAgB;gBACxC,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,KAAK,EAAE,OAAO,CAAC,MAAM;aACtB,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,UAAU;YACR,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,OAAoC;YAC9C,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YACD,MAAM,GAAG,IAAI,CAAC;YAEd,IAAI,OAAO,EAAE,YAAY,KAAK,KAAK,EAAE,CAAC;gBACpC,mBAAmB,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type AIDriver, type PromptCacheController } from '@modular-prompt/driver';
|
|
2
2
|
export interface MlxExtractRuntimeOptions {
|
|
3
|
-
model
|
|
3
|
+
/** MLX model ID or alias in models.yaml. Omitted uses the resolved default. */
|
|
4
|
+
model?: string;
|
|
4
5
|
/** Fixed cache directory. When omitted, a managed temp directory is used. */
|
|
5
6
|
cacheDir?: string;
|
|
6
7
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-mlx-extract-runtime.d.ts","sourceRoot":"","sources":["../src/create-mlx-extract-runtime.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"create-mlx-extract-runtime.d.ts","sourceRoot":"","sources":["../src/create-mlx-extract-runtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,QAAQ,EACb,KAAK,qBAAqB,EAC3B,MAAM,wBAAwB,CAAC;AAGhC,MAAM,WAAW,wBAAwB;IACvC,+EAA+E;IAC/E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6EAA6E;IAC7E,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,QAAQ,CAAC;IACjB,eAAe,EAAE,qBAAqB,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,yFAAyF;IACzF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAyBD,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,iBAAiB,CAAC,CA8B5B"}
|