@modular-prompt/extract 0.3.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -39
- package/dist/cache-lifecycle.d.ts +3 -1
- package/dist/cache-lifecycle.d.ts.map +1 -1
- package/dist/cache-lifecycle.js +21 -0
- package/dist/cache-lifecycle.js.map +1 -1
- package/dist/cli/add-command.d.ts.map +1 -1
- package/dist/cli/add-command.js +2 -1
- package/dist/cli/add-command.js.map +1 -1
- package/dist/cli/args.d.ts +3 -1
- package/dist/cli/args.d.ts.map +1 -1
- package/dist/cli/args.js +15 -1
- package/dist/cli/args.js.map +1 -1
- package/dist/cli/create-command.d.ts +2 -0
- package/dist/cli/create-command.d.ts.map +1 -1
- package/dist/cli/create-command.js +12 -5
- package/dist/cli/create-command.js.map +1 -1
- package/dist/cli/extract-command.d.ts.map +1 -1
- package/dist/cli/extract-command.js +9 -3
- package/dist/cli/extract-command.js.map +1 -1
- package/dist/cli/list-command.d.ts +1 -0
- package/dist/cli/list-command.d.ts.map +1 -1
- package/dist/cli/list-command.js +3 -1
- package/dist/cli/list-command.js.map +1 -1
- package/dist/cli/manifest.d.ts +11 -0
- package/dist/cli/manifest.d.ts.map +1 -1
- package/dist/cli/manifest.js +16 -0
- package/dist/cli/manifest.js.map +1 -1
- package/dist/cli/store.js +1 -1
- package/dist/cli/store.js.map +1 -1
- package/dist/cli.js +18 -14
- package/dist/cli.js.map +1 -1
- package/dist/create-extract-runtime.d.ts +22 -0
- package/dist/create-extract-runtime.d.ts.map +1 -0
- package/dist/create-extract-runtime.js +27 -0
- package/dist/create-extract-runtime.js.map +1 -0
- package/dist/create-extract-session.d.ts.map +1 -1
- package/dist/create-extract-session.js +4 -1
- package/dist/create-extract-session.js.map +1 -1
- package/dist/create-mlx-extract-runtime.d.ts +18 -7
- package/dist/create-mlx-extract-runtime.d.ts.map +1 -1
- package/dist/create-mlx-extract-runtime.js +11 -1
- package/dist/create-mlx-extract-runtime.js.map +1 -1
- package/dist/create-pytorch-extract-runtime.d.ts +13 -0
- package/dist/create-pytorch-extract-runtime.d.ts.map +1 -0
- package/dist/create-pytorch-extract-runtime.js +52 -0
- package/dist/create-pytorch-extract-runtime.js.map +1 -0
- package/dist/default-models.d.ts +3 -8
- package/dist/default-models.d.ts.map +1 -1
- package/dist/default-models.js +4 -16
- package/dist/default-models.js.map +1 -1
- package/dist/extract-runtime-types.d.ts +16 -0
- package/dist/extract-runtime-types.d.ts.map +1 -0
- package/dist/extract-runtime-types.js +2 -0
- package/dist/extract-runtime-types.js.map +1 -0
- package/dist/extract-store.d.ts +21 -2
- package/dist/extract-store.d.ts.map +1 -1
- package/dist/extract-store.js +49 -10
- package/dist/extract-store.js.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/model-resolution.d.ts +15 -5
- package/dist/model-resolution.d.ts.map +1 -1
- package/dist/model-resolution.js +87 -27
- package/dist/model-resolution.js.map +1 -1
- package/dist/types.d.ts +6 -0
- package/dist/types.d.ts.map +1 -1
- package/docs/API.md +316 -0
- package/docs/CACHE_DESIGN.md +475 -0
- package/docs/LOCAL_MODEL_SETUP.md +765 -0
- package/docs/PROMPT_MODULE_SPEC.md +482 -0
- package/package.json +8 -6
- /package/bin/{modular-extract.js → modular-prompt-extract.js} +0 -0
package/README.md
CHANGED
|
@@ -39,7 +39,7 @@ base (+ domain) + corpus (materials / messages) + request (inputs) ← cue
|
|
|
39
39
|
|
|
40
40
|
`type`・`cacheHint`・`partOf` は正規化層が付与する。呼び出し側で指定する必要はない。
|
|
41
41
|
|
|
42
|
-
## クイックスタート(MLX)
|
|
42
|
+
## クイックスタート(MLX / PyTorch)
|
|
43
43
|
|
|
44
44
|
### CLI
|
|
45
45
|
|
|
@@ -49,37 +49,45 @@ base (+ domain) + corpus (materials / messages) + request (inputs) ← cue
|
|
|
49
49
|
pnpm --filter @modular-prompt/extract build
|
|
50
50
|
|
|
51
51
|
# 1. 入力ファイルから meeting store を作成(デフォルト: ~/.modular-prompt/extract-cache)
|
|
52
|
-
node packages/extract/bin/modular-extract.js create meeting -m 'your-mlx-model' docs/*.txt
|
|
52
|
+
node packages/extract/bin/modular-prompt-extract.js create meeting -m 'your-mlx-model' docs/*.txt
|
|
53
|
+
|
|
54
|
+
# PyTorch モデルを使う場合は provider を明示できる(models.yaml の provider でも自動判定)
|
|
55
|
+
node packages/extract/bin/modular-prompt-extract.js create meeting-pytorch \
|
|
56
|
+
-m 'local-pytorch' --provider pytorch docs/*.txt
|
|
53
57
|
|
|
54
58
|
# 2. 既存 store にファイルを追加(incremental prefill)
|
|
55
|
-
node packages/extract/bin/modular-extract.js add meeting docs/day2.txt
|
|
59
|
+
node packages/extract/bin/modular-prompt-extract.js add meeting docs/day2.txt
|
|
56
60
|
|
|
57
61
|
# 3. 抽出クエリ(cue)を実行 — 結果は stdout
|
|
58
|
-
node packages/extract/bin/modular-extract.js extract meeting '登場人物を列挙'
|
|
62
|
+
node packages/extract/bin/modular-prompt-extract.js extract meeting '登場人物を列挙'
|
|
59
63
|
|
|
60
64
|
# 4. コンテナ内の store を一覧表示
|
|
61
|
-
node packages/extract/bin/modular-extract.js list
|
|
65
|
+
node packages/extract/bin/modular-prompt-extract.js list
|
|
62
66
|
|
|
63
67
|
# 5. store 単位のキャッシュ削除
|
|
64
|
-
node packages/extract/bin/modular-extract.js clean meeting
|
|
68
|
+
node packages/extract/bin/modular-prompt-extract.js clean meeting
|
|
65
69
|
```
|
|
66
70
|
|
|
67
71
|
| コマンド | 説明 |
|
|
68
72
|
|---------|------|
|
|
69
|
-
| `create <storename> [-d <container>] [-m <alias-or-model-id>] [files...]` | corpus を読み込み KV cache
|
|
73
|
+
| `create <storename> [-d <container>] [-m <alias-or-model-id>] [--provider <mlx\|pytorch>] [files...]` | corpus を読み込み KV cache を準備。provider と model を含む `manifest.json` を `<container>/<storename>/` に保存 |
|
|
70
74
|
| `add <storename> [-d <container>] [files...]` | 既存 store の corpus にファイルを追記し、KV cache を incremental prefill で拡張 |
|
|
71
75
|
| `extract <storename> [-d <container>] [query...]` | 指定 store のキャッシュ済み corpus に対して抽出。query が cue になる |
|
|
72
76
|
| `list [-d <container>]` | コンテナ内の全 store と manifest/KV のサマリを表示 |
|
|
73
77
|
| `clean <storename> [-d <container>]` | 指定 store の manifest + KV キャッシュを再帰削除。存在しない store は no-op |
|
|
74
78
|
| `clean --all [-d <container>]` | コンテナ全体を再帰削除。存在しないコンテナは no-op |
|
|
75
79
|
| `extract <storename> --max-tokens <n>` | 最大生成トークン数(デフォルト: 8000) |
|
|
76
|
-
| `--
|
|
80
|
+
| `--provider <mlx\|pytorch>` | create で provider を明示。省略時は models.yaml / model alias から解決 |
|
|
81
|
+
| `--dry-run` | driver を起動せず、compile 済みプロンプト全文を stdout に出力 |
|
|
77
82
|
|
|
78
83
|
`list` は各 store の model、materials 数・タイトル、作成日時、KV cache の有無を表示します。
|
|
79
84
|
|
|
85
|
+
CLI の `create` / `add` は入力ファイルを UTF-8 の文字列として読み込みます。画像ファイルを `Attachment` に変換する機能はなく、CLI の画像 material は Phase 3 の対象外です。画像を corpus に含める場合は library API の `MaterialInput.content: Attachment[]` を使用してください。MLX VLM の画像入力・画像 cache が受け付けるのは local file path のみで、URL や data URI は未対応です。
|
|
86
|
+
|
|
80
87
|
```text
|
|
81
88
|
Store: meeting
|
|
82
89
|
Model: mlx-community/SomeModel-4bit
|
|
90
|
+
Provider: mlx
|
|
83
91
|
Materials: 2 (notes.txt, meeting.md)
|
|
84
92
|
Created: 2026-09-07T03:00:00.000Z
|
|
85
93
|
Updated: 2026-09-07T04:00:00.000Z
|
|
@@ -88,17 +96,26 @@ Store: meeting
|
|
|
88
96
|
|
|
89
97
|
```bash
|
|
90
98
|
# プロンプト確認(create)
|
|
91
|
-
modular-extract create meeting --dry-run docs/notes.txt
|
|
99
|
+
modular-prompt-extract create meeting --dry-run docs/notes.txt
|
|
92
100
|
|
|
93
101
|
# プロンプト確認(extract — store の manifest が必要)
|
|
94
|
-
modular-extract extract meeting --dry-run '登場人物を列挙'
|
|
102
|
+
modular-prompt-extract extract meeting --dry-run '登場人物を列挙'
|
|
95
103
|
```
|
|
96
104
|
|
|
97
105
|
`-d` 省略時のデフォルトは `~/.modular-prompt/extract-cache` で、create/add/extract/list/clean 共通の **store コンテナ**を指定します。`MODULAR_PROMPT_HOME` を設定している場合は、その値の下の `extract-cache` が使用されます。`-m` には models.yaml の alias(例: `default`)または生の HF model ID を指定できます。create/add/extract/clean では `<storename>` が必須(`clean --all` を除く)で、コンテナ配下の `<storename>/` が利用されます。
|
|
98
|
-
`-m` 省略時は、同梱 models 設定と `~/.modular-prompt/models.yaml`(`MODULAR_PROMPT_HOME` で変更可)をマージし、`models.default`、なければ先頭のモデルを使用します。user yaml の `default` は同梱 default を上書きします。
|
|
99
|
-
`MLX_MODEL` 環境変数も後方互換のためサポートしており、設定時は同梱 default のモデル ID として扱います。user yaml の `models.default` は `MLX_MODEL` より優先されます。
|
|
100
106
|
|
|
101
|
-
|
|
107
|
+
### models.yaml 連携
|
|
108
|
+
|
|
109
|
+
マシン共通のモデル定義は **`~/.modular-prompt/models.yaml`**(単一ファイル。`models/` ディレクトリは非対応)に置きます。`MODULAR_PROMPT_HOME` でホームディレクトリを変更できます。
|
|
110
|
+
|
|
111
|
+
| 順位 | ソース | 説明 |
|
|
112
|
+
|------|--------|------|
|
|
113
|
+
| 1 | CLI `-m` | 最優先。alias または生の HF model ID |
|
|
114
|
+
| 2 | user yaml の `models.default` | `-m` 省略時のみ使用。同梱 fallback や先頭エントリ自動選択はなし |
|
|
115
|
+
|
|
116
|
+
`-m` を省略した場合は、`models.default` が user yaml に明示されているときだけそのモデルを使います。未設定時は driver を起動せず、`-m` 指定または `models.default` 定義を案内するエラーを返します。モデル alias に `provider: mlx` または `provider: pytorch` を設定すると provider を自動判定できます。provider を推論できない生 model ID は、`create --provider mlx` / `create --provider pytorch` のように明示してください。create は解決後の生 model ID と provider を store の `manifest.json` に保存し、以降の add/extract は manifest の provider + model を検証して同じ cache runtime を使います。provider の異なる store は cache 形式が非互換のため開けません。
|
|
117
|
+
|
|
118
|
+
ローカルテスト用のモデルは `~/.modular-prompt/models.testing.yaml` に分けて置き、手元の extract 実行では `MODULAR_PROMPT_MODELS_PROFILE=testing` を指定できます。設定ファイルのサンプルと統合テストの convention alias は [ローカルモデルセットアップガイド](./docs/LOCAL_MODEL_SETUP.md) を参照してください。
|
|
102
119
|
|
|
103
120
|
たとえば `~/.modular-prompt/models.yaml` に次を置くと、`create meeting -m default` と `create meeting` の両方でこのモデルが選ばれます。
|
|
104
121
|
|
|
@@ -109,9 +126,29 @@ models:
|
|
|
109
126
|
model: mlx-community/YourModel-4bit
|
|
110
127
|
```
|
|
111
128
|
|
|
129
|
+
PyTorch (Transformers) の text-only extract は、次のように `driverOptions.device` を設定できます。`venvPath` は外部 venv を使う場合だけ指定し、省略時は driver 管理の既定 runtime を使用します。CUDA の場合は driver 側の runtime 構成に従います。
|
|
130
|
+
|
|
131
|
+
```yaml
|
|
132
|
+
models:
|
|
133
|
+
local-pytorch:
|
|
134
|
+
provider: pytorch
|
|
135
|
+
model: meta-llama/Llama-3.2-3B-Instruct
|
|
136
|
+
driverOptions:
|
|
137
|
+
device: cuda
|
|
138
|
+
```
|
|
139
|
+
|
|
112
140
|
モデルが設定されていない構成では、`-m <model-id-or-alias>` を指定するか、user yaml に `models.default` を定義してください。
|
|
113
141
|
|
|
114
|
-
**MLX バックエンドは mlx-lm
|
|
142
|
+
**MLX バックエンドは models.yaml の指定に従う**。未指定時は `auto` で、モデル種別に応じて `mlx-lm` / `mlx-vlm` を選択する。`backend: 'vlm'` を指定した VLM 判定モデルは、画像なしの text-only exact KV cache に加えて、画像 material を含む prompt の vision cache もディスクへ保存して extract session 間で再利用できる。画像あり VLM cache は text-only VLM / LM の store とは別 namespace・非互換で、VLM の incremental prefill は対象外。PyTorch runtime は現状 text-only で、MLX の `backend` / `maxImageSize` は無視されます。
|
|
143
|
+
|
|
144
|
+
```yaml
|
|
145
|
+
models:
|
|
146
|
+
default:
|
|
147
|
+
provider: mlx
|
|
148
|
+
model: mlx-community/Your-VLM-4bit
|
|
149
|
+
driverOptions:
|
|
150
|
+
backend: vlm # 省略時は auto
|
|
151
|
+
```
|
|
115
152
|
|
|
116
153
|
### ライブラリ API
|
|
117
154
|
|
|
@@ -119,10 +156,11 @@ models:
|
|
|
119
156
|
import {
|
|
120
157
|
createExtractSession,
|
|
121
158
|
createMlxExtractRuntime,
|
|
159
|
+
createPytorchExtractRuntime,
|
|
122
160
|
} from '@modular-prompt/extract';
|
|
123
161
|
|
|
124
162
|
const runtime = await createMlxExtractRuntime({
|
|
125
|
-
model: '
|
|
163
|
+
model: 'your-mlx-model',
|
|
126
164
|
});
|
|
127
165
|
|
|
128
166
|
try {
|
|
@@ -149,6 +187,11 @@ try {
|
|
|
149
187
|
} finally {
|
|
150
188
|
await runtime.close();
|
|
151
189
|
}
|
|
190
|
+
|
|
191
|
+
// PyTorch の場合も同じ ExtractSession API を使う
|
|
192
|
+
const pytorchRuntime = await createPytorchExtractRuntime({ model: 'local-pytorch' });
|
|
193
|
+
// pytorchRuntime.driver / pytorchRuntime.cacheController を同じように session へ渡す
|
|
194
|
+
await pytorchRuntime.close();
|
|
152
195
|
```
|
|
153
196
|
|
|
154
197
|
## キャッシュの意図と制約
|
|
@@ -159,13 +202,13 @@ try {
|
|
|
159
202
|
|-----------|-------------|
|
|
160
203
|
| `session.close()`(デフォルト) | handle を `release` マーク → 次の `runtime.close()` で **KV ファイル削除** |
|
|
161
204
|
| `session.close({ releaseCache: false })` | release しない → **KV ファイルは disk に残る**(CLI はこちら) |
|
|
162
|
-
| `runtime.close()`(固定 cacheDir) | `release` 済みエントリの
|
|
205
|
+
| `runtime.close()`(固定 cacheDir) | `release` 済みエントリの LM/VLM cache files を削除 |
|
|
163
206
|
| `runtime.close()`(一時 cacheDir) | **ディレクトリごと削除** |
|
|
164
207
|
| `add <storename> files...` | 既存 store を staging にコピーし、必須 cache prepare(空 handle は失敗)と manifest 更新が成功した後に入れ替え |
|
|
165
208
|
| `clean <storename> [-d <container>]` | 1 store の manifest + KV キャッシュを再帰削除 |
|
|
166
209
|
| `clean --all [-d <container>]` | コンテナ内の全 store を再帰削除 |
|
|
167
210
|
|
|
168
|
-
`create` 直後に store 内へ `manifest.json` だけ残って
|
|
211
|
+
`create` 直後に store 内へ `manifest.json` だけ残って cache file が無い場合、以前のバージョンでは `session.close()` が release していたのが原因。CLI は `releaseCache: false` で修正済み。
|
|
169
212
|
|
|
170
213
|
### 意図
|
|
171
214
|
|
|
@@ -183,9 +226,9 @@ try {
|
|
|
183
226
|
| driver / cacheController の終了 | 呼び出し側の責務(`runtime.close()` 等) |
|
|
184
227
|
| セッション終了 | `session.close()` — デフォルトで handle `release()`。固定 cacheDir を残す場合は `{ releaseCache: false }` |
|
|
185
228
|
|
|
186
|
-
`cacheController` は **必須**。`createMlxExtractRuntime` の `model` は省略でき、CLI と同じ models.yaml
|
|
229
|
+
`cacheController` は **必須**。`createMlxExtractRuntime` / `createPytorchExtractRuntime` の `model` は省略でき、CLI と同じ user models.yaml の `models.default` 解決を行います。provider を明示的に選ぶ場合は `createExtractRuntime({ model, provider })` を使えます。モデル設定がない場合や provider を推論できない生 ID はエラーになります。PyTorch runtime は text-only で、`driverOptions.device` / `venvPath` は models.yaml から渡されます。固定 `cacheDir` は PyTorch の runtime variant / device ごとに分け、CPU と CUDA などで共有しないでください(manifest は provider と model を検証します)。キャッシュ非対応モードは提供しない。
|
|
187
230
|
|
|
188
|
-
詳細は [プロンプトキャッシュ設計](
|
|
231
|
+
詳細は [プロンプトキャッシュ設計](./docs/CACHE_DESIGN.md) および [API 仕様](./docs/API.md) を参照。
|
|
189
232
|
|
|
190
233
|
## サンプル
|
|
191
234
|
|
|
@@ -296,8 +339,10 @@ console.log(result.structured); // schema に沿った JSON
|
|
|
296
339
|
|---------|------|
|
|
297
340
|
| `createExtractSession` | 抽出セッションを生成 |
|
|
298
341
|
| `createMlxExtractRuntime` | MLX 用 driver + cacheController バンドル |
|
|
342
|
+
| `createPytorchExtractRuntime` | PyTorch 用 driver + cacheController バンドル(text-only) |
|
|
343
|
+
| `createExtractRuntime` | 解決済み provider に応じた runtime factory |
|
|
299
344
|
| `resolveModelSpec` | alias または生 model ID から extract 用 ModelSpec を解決 |
|
|
300
|
-
| `createDriver` | 解決済み ModelSpec から AIService 経由で MLX driver を生成 |
|
|
345
|
+
| `createDriver` | 解決済み ModelSpec から AIService 経由で MLX / PyTorch driver を生成 |
|
|
301
346
|
| `resolveDefaultContainerDir` | `MODULAR_PROMPT_HOME` に基づくデフォルト cache container を解決 |
|
|
302
347
|
| `resolveStoreDir` | cache コンテナと storename から store ディレクトリを解決 |
|
|
303
348
|
| `validateStorename` | storename の形式と予約語を検証 |
|
|
@@ -308,32 +353,32 @@ console.log(result.structured); // schema に沿った JSON
|
|
|
308
353
|
|
|
309
354
|
型: `ExtractCorpus`, `ExtractRequest`, `ExtractResult`, `ExtractSession`, `MaterialInput`, `MessageInput`, `ChunkInput` など。
|
|
310
355
|
|
|
311
|
-
完全な API リファレンスは [API.md](./API.md) を参照。
|
|
356
|
+
完全な API リファレンスは [API.md](./docs/API.md) を参照。
|
|
312
357
|
|
|
313
|
-
## CLI(`modular-extract`)
|
|
358
|
+
## CLI(`modular-prompt-extract`)
|
|
314
359
|
|
|
315
|
-
`bin/modular-extract.js` 経由で利用できる簡易 CLI。
|
|
360
|
+
`bin/modular-prompt-extract.js` 経由で利用できる簡易 CLI。
|
|
316
361
|
|
|
317
362
|
```bash
|
|
318
363
|
pnpm --filter @modular-prompt/extract build
|
|
319
364
|
|
|
320
|
-
modular-extract create meeting [-d <cache-dir>] [-m <alias-or-model-id>] file1.txt file2.txt
|
|
321
|
-
modular-extract add meeting [-d <cache-dir>] file3.txt
|
|
322
|
-
modular-extract create contract [-d <cache-dir>] [-m <alias-or-model-id>] contract.pdf
|
|
323
|
-
modular-extract extract meeting [-d <cache-dir>] '抽出したい内容の指示'
|
|
324
|
-
modular-extract extract contract [-d <cache-dir>] '契約期間を抽出'
|
|
325
|
-
modular-extract list [-d <cache-dir>]
|
|
326
|
-
modular-extract clean meeting [-d <cache-dir>]
|
|
327
|
-
modular-extract clean --all [-d <cache-dir>]
|
|
365
|
+
modular-prompt-extract create meeting [-d <cache-dir>] [-m <alias-or-model-id>] [--provider <mlx|pytorch>] file1.txt file2.txt
|
|
366
|
+
modular-prompt-extract add meeting [-d <cache-dir>] file3.txt
|
|
367
|
+
modular-prompt-extract create contract [-d <cache-dir>] [-m <alias-or-model-id>] contract.pdf
|
|
368
|
+
modular-prompt-extract extract meeting [-d <cache-dir>] '抽出したい内容の指示'
|
|
369
|
+
modular-prompt-extract extract contract [-d <cache-dir>] '契約期間を抽出'
|
|
370
|
+
modular-prompt-extract list [-d <cache-dir>]
|
|
371
|
+
modular-prompt-extract clean meeting [-d <cache-dir>]
|
|
372
|
+
modular-prompt-extract clean --all [-d <cache-dir>]
|
|
328
373
|
```
|
|
329
374
|
|
|
330
|
-
`<storename>` は create/add/extract/clean の positional 第1引数で必須です(`clean --all` を除く)。`[a-zA-Z0-9][a-zA-Z0-9_-]*` に一致し、`create`・`add`・`extract`・`list`・`clean` は使用できません。`-d` は store コンテナを指定し、create は `<container>/<storename>/` にキャッシュと `manifest.json` を保存します。既存 store に対する create は失敗するため、`modular-extract clean <storename>`(必要に応じて `-d <container>`)で削除してから再実行します。
|
|
375
|
+
`<storename>` は create/add/extract/clean の positional 第1引数で必須です(`clean --all` を除く)。`[a-zA-Z0-9][a-zA-Z0-9_-]*` に一致し、`create`・`add`・`extract`・`list`・`clean` は使用できません。`-d` は store コンテナを指定し、create は `<container>/<storename>/` にキャッシュと `manifest.json` を保存します。既存 store に対する create は失敗するため、`modular-prompt-extract clean <storename>`(必要に応じて `-d <container>`)で削除してから再実行します。
|
|
331
376
|
|
|
332
|
-
`add <storename> files...` は manifest の model
|
|
377
|
+
`add <storename> files...` は manifest の provider + model を検証して既存 store に資料を追加します。新しいファイルは絶対パスを `id` として追記され、同じ `id`・同じ内容の再追加はスキップされます。同じ `id` の内容が変わっている場合は、キャッシュとの不整合を避けるためエラーになります。その場合は `clean` してから `create` し直してください。`add --dry-run` は driver を起動せず、マージ後のプロンプトを表示します。
|
|
333
378
|
|
|
334
379
|
`add` は既存 store を直接上書きしません。staging store で必須 cache prepare と manifest 書き込みを完了してから store ディレクトリを入れ替えるため、空 handle を含む prefill の失敗、または manifest 更新の失敗時は既存の corpus と KV cache が保持されます。通常の `createExtractSession` / `extract` は引き続き cache prepare の失敗を best-effort で扱います。
|
|
335
380
|
|
|
336
|
-
`-m` は models.yaml の alias(`default` など)または生の HF model ID
|
|
381
|
+
`-m` は models.yaml の alias(`default` など)または生の HF model ID を受け付けます。省略時は user の `~/.modular-prompt/models.yaml` にある `models.default` から解決します。モデルが未設定の場合は明示的な `-m` または `models.default` が必要です。生 model ID の provider を推論できない場合は `--provider mlx` / `--provider pytorch` を指定してください。`create` は解決後の生 model ID と provider を store 内の `manifest.json` に保存し、`extract` と `add` は manifest の provider + model を検証して再開します。backend がない既存 MLX manifest は `auto` として扱うため、従来どおりモデル種別の自動判定になります。VLM は text-only cache と画像 material 用の vision cache を別 namespace で使用し、VLM incremental prefill は対象外です。PyTorch は現状 text-only です。
|
|
337
382
|
|
|
338
383
|
### 旧 CLI / キャッシュレイアウトからの移行
|
|
339
384
|
|
|
@@ -345,7 +390,7 @@ mkdir -p ~/.modular-prompt/extract-cache
|
|
|
345
390
|
mv ./.extract-cache/meeting ~/.modular-prompt/extract-cache/
|
|
346
391
|
```
|
|
347
392
|
|
|
348
|
-
移行後は新形式で `modular-extract extract meeting '...'` を実行します。複数の store がある場合は、それぞれ移動してください。
|
|
393
|
+
移行後は新形式で `modular-prompt-extract extract meeting '...'` を実行します。複数の store がある場合は、それぞれ移動してください。
|
|
349
394
|
|
|
350
395
|
#353 より前の flat レイアウト(コンテナ直下の `manifest.json` と cache files)を使用していた場合は、storename を決めて次のように移行します。
|
|
351
396
|
|
|
@@ -369,8 +414,8 @@ MLX 統合テストは macOS + MLX 設定がある環境でのみ実行される
|
|
|
369
414
|
|
|
370
415
|
## 関連ドキュメント
|
|
371
416
|
|
|
372
|
-
- [API 仕様](./API.md)
|
|
373
|
-
- [プロンプトモジュール仕様](
|
|
374
|
-
- [プロンプトキャッシュ設計](
|
|
375
|
-
- [ローカルモデルセットアップ](
|
|
417
|
+
- [API 仕様](./docs/API.md)
|
|
418
|
+
- [プロンプトモジュール仕様](./docs/PROMPT_MODULE_SPEC.md)
|
|
419
|
+
- [プロンプトキャッシュ設計](./docs/CACHE_DESIGN.md)
|
|
420
|
+
- [ローカルモデルセットアップ](./docs/LOCAL_MODEL_SETUP.md)
|
|
376
421
|
- 親 Issue: [#330](https://github.com/otolab/modular-prompt/issues/330)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { PromptModule } from '@modular-prompt/core';
|
|
2
|
-
import type
|
|
2
|
+
import { type AIDriver, type CacheHandle, type PromptCacheController } from '@modular-prompt/driver';
|
|
3
3
|
import type { ExtractCorpus, ExtractRequest } from './types.js';
|
|
4
4
|
export interface CacheLifecycleState {
|
|
5
5
|
handle: CacheHandle | null;
|
|
@@ -8,6 +8,8 @@ export interface CacheLifecycleState {
|
|
|
8
8
|
export interface PrepareSessionCacheOptions {
|
|
9
9
|
/** Reject an empty cache handle instead of falling back to an uncached query. */
|
|
10
10
|
required?: boolean;
|
|
11
|
+
/** Image resize limit used by a VLM cache prefill. */
|
|
12
|
+
maxImageSize?: number;
|
|
11
13
|
}
|
|
12
14
|
export declare function ensureCacheControllerReady(driver: AIDriver, state: CacheLifecycleState): Promise<void>;
|
|
13
15
|
export declare function prepareSessionCache<TContext>(cacheController: PromptCacheController, model: string, sessionBaseModule: PromptModule<TContext>, corpus: ExtractCorpus, request: ExtractRequest, baseModule: PromptModule<TContext> | undefined, state: CacheLifecycleState, options?: PrepareSessionCacheOptions): Promise<CacheHandle | null>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache-lifecycle.d.ts","sourceRoot":"","sources":["../src/cache-lifecycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"cache-lifecycle.d.ts","sourceRoot":"","sources":["../src/cache-lifecycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAEL,KAAK,QAAQ,EACb,KAAK,WAAW,EAChB,KAAK,qBAAqB,EAC3B,MAAM,wBAAwB,CAAC;AAGhC,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAGhE,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IAC3B,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,0BAA0B;IACzC,iFAAiF;IACjF,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,sDAAsD;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAkBD,wBAAsB,0BAA0B,CAC9C,MAAM,EAAE,QAAQ,EAChB,KAAK,EAAE,mBAAmB,GACzB,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,mBAAmB,CAAC,QAAQ,EAChD,eAAe,EAAE,qBAAqB,EACtC,KAAK,EAAE,MAAM,EACb,iBAAiB,EAAE,YAAY,CAAC,QAAQ,CAAC,EACzC,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,cAAc,EACvB,UAAU,EAAE,YAAY,CAAC,QAAQ,CAAC,GAAG,SAAS,EAC9C,KAAK,EAAE,mBAAmB,EAC1B,OAAO,GAAE,0BAA+B,GACvC,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CA0C7B;AAED,wBAAgB,mBAAmB,CACjC,eAAe,EAAE,qBAAqB,EACtC,KAAK,EAAE,mBAAmB,GACzB,IAAI,CAKN"}
|
package/dist/cache-lifecycle.js
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
|
+
import { extractImagePaths, } from '@modular-prompt/driver';
|
|
1
2
|
import { partitionPrompt } from '@modular-prompt/driver';
|
|
2
3
|
import { compileExtractPrompt } from './compile-extract-prompt.js';
|
|
4
|
+
function extractImagesFromElements(elements) {
|
|
5
|
+
return elements.flatMap((element) => {
|
|
6
|
+
switch (element.type) {
|
|
7
|
+
case 'text':
|
|
8
|
+
return [];
|
|
9
|
+
case 'message':
|
|
10
|
+
return element.role === 'tool' ? [] : extractImagePaths(element.content);
|
|
11
|
+
case 'material':
|
|
12
|
+
case 'chunk':
|
|
13
|
+
return extractImagePaths(element.content);
|
|
14
|
+
default:
|
|
15
|
+
return [];
|
|
16
|
+
}
|
|
17
|
+
});
|
|
18
|
+
}
|
|
3
19
|
export async function ensureCacheControllerReady(driver, state) {
|
|
4
20
|
if (state.controllerReady) {
|
|
5
21
|
return;
|
|
@@ -25,6 +41,11 @@ export async function prepareSessionCache(cacheController, model, sessionBaseMod
|
|
|
25
41
|
data: cacheable.data,
|
|
26
42
|
tools: request.options?.tools,
|
|
27
43
|
reasoningEffort: request.options?.reasoningEffort,
|
|
44
|
+
images: extractImagesFromElements([
|
|
45
|
+
...cacheable.instructions,
|
|
46
|
+
...cacheable.data,
|
|
47
|
+
]),
|
|
48
|
+
maxImageSize: options.maxImageSize,
|
|
28
49
|
});
|
|
29
50
|
if (!newHandle.ref) {
|
|
30
51
|
if (options.required) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache-lifecycle.js","sourceRoot":"","sources":["../src/cache-lifecycle.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cache-lifecycle.js","sourceRoot":"","sources":["../src/cache-lifecycle.ts"],"names":[],"mappings":"AACA,OAAO,EACL,iBAAiB,GAIlB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAcnE,SAAS,yBAAyB,CAAC,QAA4B;IAC7D,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAClC,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,MAAM;gBACT,OAAO,EAAE,CAAC;YACZ,KAAK,SAAS;gBACZ,OAAO,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC3E,KAAK,UAAU,CAAC;YAChB,KAAK,OAAO;gBACV,OAAO,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC5C;gBACE,OAAO,EAAE,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,MAAgB,EAChB,KAA0B;IAE1B,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;QAC1B,OAAO;IACT,CAAC;IAED,IAAI,iBAAiB,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,eAAe,KAAK,UAAU,EAAE,CAAC;QAChF,MAAM,MAAM,CAAC,eAAe,EAAE,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,eAAsC,EACtC,KAAa,EACb,iBAAyC,EACzC,MAAqB,EACrB,OAAuB,EACvB,UAA8C,EAC9C,KAA0B,EAC1B,UAAsC,EAAE;IAExC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,iBAAiB,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACtF,MAAM,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,mBAAmB,GACvB,SAAS,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAEjE,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;QACxF,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC;QAC9C,KAAK;QACL,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK;QAC7B,eAAe,EAAE,OAAO,CAAC,OAAO,EAAE,eAAe;QACjD,MAAM,EAAE,yBAAyB,CAAC;YAChC,GAAG,SAAS,CAAC,YAAY;YACzB,GAAG,SAAS,CAAC,IAAI;SAClB,CAAC;QACF,YAAY,EAAE,OAAO,CAAC,YAAY;KACnC,CAAC,CAAC;IAEH,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;QACnB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CACb,8EAA8E,CAC/E,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;QACpB,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,UAAU,KAAK,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACnE,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IAChD,OAAO,KAAK,CAAC,MAAM,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,eAAsC,EACtC,KAA0B;IAE1B,IAAI,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC;QACtB,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1C,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;IACtB,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"add-command.d.ts","sourceRoot":"","sources":["../../src/cli/add-command.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,cAAc,EAEf,MAAM,qBAAqB,CAAC;AAM7B,MAAM,WAAW,iBAAiB;IAChC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"add-command.d.ts","sourceRoot":"","sources":["../../src/cli/add-command.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,cAAc,EAEf,MAAM,qBAAqB,CAAC;AAM7B,MAAM,WAAW,iBAAiB;IAChC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAwBtF;AAID,OAAO,EAAE,cAAc,EAAE,CAAC"}
|
package/dist/cli/add-command.js
CHANGED
|
@@ -21,7 +21,8 @@ export async function runAddCommand(options) {
|
|
|
21
21
|
existingManifest: manifest,
|
|
22
22
|
});
|
|
23
23
|
console.error(`Cache extended: ${storeDir}`);
|
|
24
|
-
console.error(`Materials: ${result.manifest.materials.length} file(s), model: ${result.model}`
|
|
24
|
+
console.error(`Materials: ${result.manifest.materials.length} file(s), model: ${result.model}, provider: ${result.provider}`
|
|
25
|
+
+ (result.backend !== undefined ? `, backend: ${result.backend}` : ''));
|
|
25
26
|
}
|
|
26
27
|
// Keep the merge operation discoverable next to the CLI entry point while the
|
|
27
28
|
// implementation remains in the reusable extract-store layer.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"add-command.js","sourceRoot":"","sources":["../../src/cli/add-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,wBAAwB,GACzB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAU7C,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAA0B;IAC5D,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC7E,MAAM,iBAAiB,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC;IAE3C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,mBAAmB,CAAC,EAAE,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC;QACxC,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,iBAAiB;QACjB,gBAAgB,EAAE,QAAQ;KAC3B,CAAC,CAAC;IAEH,OAAO,CAAC,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,KAAK,
|
|
1
|
+
{"version":3,"file":"add-command.js","sourceRoot":"","sources":["../../src/cli/add-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EACL,oBAAoB,EACpB,cAAc,EACd,wBAAwB,GACzB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAU7C,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAA0B;IAC5D,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC7E,MAAM,iBAAiB,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC;IAE3C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,mBAAmB,CAAC,EAAE,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC;QACxC,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,iBAAiB;QACjB,gBAAgB,EAAE,QAAQ;KAC3B,CAAC,CAAC;IAEH,OAAO,CAAC,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,KAAK,CACX,cAAc,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,oBAAoB,MAAM,CAAC,KAAK,eAAe,MAAM,CAAC,QAAQ,EAAE;UAC1G,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACzE,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,8DAA8D;AAC9D,OAAO,EAAE,cAAc,EAAE,CAAC"}
|
package/dist/cli/args.d.ts
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
|
+
import type { ExtractProvider } from '../extract-runtime-types.js';
|
|
1
2
|
export type CliCommand = 'create' | 'add' | 'extract' | 'list' | 'clean' | 'help';
|
|
2
3
|
export interface ParsedArgs {
|
|
3
4
|
command?: CliCommand;
|
|
4
5
|
cacheDir?: string;
|
|
5
6
|
model?: string;
|
|
7
|
+
provider?: ExtractProvider;
|
|
6
8
|
maxTokens?: number;
|
|
7
9
|
dryRun?: boolean;
|
|
8
10
|
all?: boolean;
|
|
9
11
|
storename?: string;
|
|
10
12
|
positional: string[];
|
|
11
13
|
}
|
|
12
|
-
/** Parse modular-extract command-line arguments. */
|
|
14
|
+
/** Parse modular-prompt-extract command-line arguments. */
|
|
13
15
|
export declare function parseArgs(argv: string[]): ParsedArgs;
|
|
14
16
|
//# sourceMappingURL=args.d.ts.map
|
package/dist/cli/args.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAElF,MAAM,WAAW,UAAU;IACzB,OAAO,CAAC,EAAE,UAAU,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,eAAe,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAiED,2DAA2D;AAC3D,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,UAAU,CA0FpD"}
|
package/dist/cli/args.js
CHANGED
|
@@ -13,6 +13,12 @@ function parseMaxTokens(value) {
|
|
|
13
13
|
}
|
|
14
14
|
return parsed;
|
|
15
15
|
}
|
|
16
|
+
function parseProvider(value) {
|
|
17
|
+
if (value === 'mlx' || value === 'pytorch') {
|
|
18
|
+
return value;
|
|
19
|
+
}
|
|
20
|
+
throw new Error('--provider must be either mlx or pytorch');
|
|
21
|
+
}
|
|
16
22
|
function validateCommandOptions(result) {
|
|
17
23
|
if (result.all && result.command !== 'clean') {
|
|
18
24
|
throw new Error('--all is only valid with clean');
|
|
@@ -23,6 +29,9 @@ function validateCommandOptions(result) {
|
|
|
23
29
|
if ((result.command === 'extract' || result.command === 'add') && result.model !== undefined) {
|
|
24
30
|
throw new Error('--model is only valid with create');
|
|
25
31
|
}
|
|
32
|
+
if (result.command !== 'create' && result.provider !== undefined) {
|
|
33
|
+
throw new Error('--provider is only valid with create');
|
|
34
|
+
}
|
|
26
35
|
if (result.command === 'list') {
|
|
27
36
|
if (result.model !== undefined || result.maxTokens !== undefined || result.dryRun) {
|
|
28
37
|
throw new Error('list does not accept create/extract options');
|
|
@@ -49,7 +58,7 @@ function validateCommandOptions(result) {
|
|
|
49
58
|
}
|
|
50
59
|
}
|
|
51
60
|
}
|
|
52
|
-
/** Parse modular-extract command-line arguments. */
|
|
61
|
+
/** Parse modular-prompt-extract command-line arguments. */
|
|
53
62
|
export function parseArgs(argv) {
|
|
54
63
|
const result = { positional: [] };
|
|
55
64
|
let index = 0;
|
|
@@ -83,6 +92,11 @@ export function parseArgs(argv) {
|
|
|
83
92
|
index += 2;
|
|
84
93
|
continue;
|
|
85
94
|
}
|
|
95
|
+
if (!optionsEnded && arg === '--provider') {
|
|
96
|
+
result.provider = parseProvider(requireOptionValue(argv, index, arg, 'mlx or pytorch'));
|
|
97
|
+
index += 2;
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
86
100
|
if (!optionsEnded && arg === '--dry-run') {
|
|
87
101
|
result.dryRun = true;
|
|
88
102
|
index += 1;
|
package/dist/cli/args.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAiB/C,SAAS,kBAAkB,CAAC,IAAc,EAAE,KAAa,EAAE,MAAc,EAAE,WAAmB;IAC5F,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,aAAa,WAAW,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAa;IACnC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC3C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAkB;IAChD,IAAI,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC7F,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAC9B,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClF,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QAC/B,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClF,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;YACtE,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;AACH,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,MAAM,MAAM,GAAe,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IAC9C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAE,CAAC;QAEzB,IAAI,CAAC,YAAY,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC;YACxD,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC;YACxB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YAClC,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,YAAY,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7D,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;gBAChG,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC;gBACrB,KAAK,IAAI,CAAC,CAAC;gBACX,SAAS;YACX,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,aAAa,CAAC,EAAE,CAAC;YAC7D,MAAM,CAAC,QAAQ,GAAG,kBAAkB,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;YACjE,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,CAAC,EAAE,CAAC;YACzD,MAAM,CAAC,KAAK,GAAG,kBAAkB,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC;YAClE,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YAC1C,MAAM,CAAC,QAAQ,GAAG,aAAa,CAAC,kBAAkB,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,gBAAgB,CAAC,CAAC,CAAC;YACxF,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;YACzC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC;YACrB,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;YACrC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC;YAClB,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,GAAG,KAAK,cAAc,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,oBAAoB,CAAC,CAAC;YACzE,MAAM,CAAC,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACzC,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5B,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;QAC1H,MAAM,CAAC,SAAS,EAAE,GAAG,UAAU,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC;QACrD,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,KAAK,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;gBACzD,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,OAAO,6CAA6C,CAAC,CAAC;YAClF,CAAC;YACD,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAC7B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;YAC7B,MAAM,CAAC,UAAU,GAAG,UAAU,CAAC;QACjC,CAAC;IACH,CAAC;IAED,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/B,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import type { ExtractProvider } from '../extract-runtime-types.js';
|
|
1
2
|
export interface CreateCommandOptions {
|
|
2
3
|
/** Container directory containing one subdirectory per store. */
|
|
3
4
|
cacheDir: string;
|
|
4
5
|
storename: string;
|
|
5
6
|
model?: string;
|
|
7
|
+
provider?: ExtractProvider;
|
|
6
8
|
files: string[];
|
|
7
9
|
dryRun?: boolean;
|
|
8
10
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-command.d.ts","sourceRoot":"","sources":["../../src/cli/create-command.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"create-command.d.ts","sourceRoot":"","sources":["../../src/cli/create-command.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAOnE,MAAM,WAAW,oBAAoB;IACnC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,eAAe,CAAC;IAC3B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAuD5F"}
|
|
@@ -16,22 +16,28 @@ export async function runCreateCommand(options) {
|
|
|
16
16
|
}
|
|
17
17
|
if (await storeExists(storeDir)) {
|
|
18
18
|
throw new Error(`Store already exists: ${storeDir}\n`
|
|
19
|
-
+ `Run \`modular-extract clean ${options.storename}\` before creating it again.`);
|
|
19
|
+
+ `Run \`modular-prompt-extract clean ${options.storename}\` before creating it again.`);
|
|
20
20
|
}
|
|
21
21
|
await mkdir(storeDir, { recursive: true });
|
|
22
22
|
let storeReady = false;
|
|
23
|
-
let
|
|
23
|
+
let prepared;
|
|
24
24
|
try {
|
|
25
|
-
|
|
25
|
+
prepared = await prepareExtractCache({
|
|
26
26
|
cacheDir: storeDir,
|
|
27
27
|
model: options.model,
|
|
28
|
+
...(options.provider !== undefined ? { provider: options.provider } : {}),
|
|
28
29
|
materials,
|
|
29
30
|
});
|
|
30
31
|
const createdAt = new Date().toISOString();
|
|
31
32
|
await writeManifest(storeDir, {
|
|
32
33
|
version: 1,
|
|
33
34
|
storename: options.storename,
|
|
34
|
-
model:
|
|
35
|
+
model: prepared.model,
|
|
36
|
+
provider: prepared.provider,
|
|
37
|
+
backend: prepared.backend,
|
|
38
|
+
...(prepared.maxImageSize !== undefined
|
|
39
|
+
? { maxImageSize: prepared.maxImageSize }
|
|
40
|
+
: {}),
|
|
35
41
|
materials,
|
|
36
42
|
createdAt,
|
|
37
43
|
updatedAt: createdAt,
|
|
@@ -44,6 +50,7 @@ export async function runCreateCommand(options) {
|
|
|
44
50
|
}
|
|
45
51
|
}
|
|
46
52
|
console.error(`Cache prepared: ${storeDir}`);
|
|
47
|
-
console.error(`Materials: ${materials.length} file(s), model: ${
|
|
53
|
+
console.error(`Materials: ${materials.length} file(s), model: ${prepared.model}, provider: ${prepared.provider}`
|
|
54
|
+
+ (prepared.backend !== undefined ? `, backend: ${prepared.backend}` : ''));
|
|
48
55
|
}
|
|
49
56
|
//# sourceMappingURL=create-command.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-command.js","sourceRoot":"","sources":["../../src/cli/create-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"create-command.js","sourceRoot":"","sources":["../../src/cli/create-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAY1D,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAA6B;IAClE,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC;IAE3C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,mBAAmB,CAAC,EAAE,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,MAAM,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,yBAAyB,QAAQ,IAAI;cACnC,sCAAsC,OAAO,CAAC,SAAS,8BAA8B,CACxF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,QAAqE,CAAC;IAC1E,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,mBAAmB,CAAC;YACnC,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,SAAS;SACV,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE3C,MAAM,aAAa,CAAC,QAAQ,EAAE;YAC5B,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,GAAG,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS;gBACrC,CAAC,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE;gBACzC,CAAC,CAAC,EAAE,CAAC;YACP,SAAS;YACT,SAAS;YACT,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;QACH,UAAU,GAAG,IAAI,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,KAAK,CACX,cAAc,SAAS,CAAC,MAAM,oBAAoB,QAAQ,CAAC,KAAK,eAAe,QAAQ,CAAC,QAAQ,EAAE;UAC9F,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAC7E,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-command.d.ts","sourceRoot":"","sources":["../../src/cli/extract-command.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extract-command.d.ts","sourceRoot":"","sources":["../../src/cli/extract-command.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,qBAAqB;IACpC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC,CA8CvF"}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { resolve } from 'node:path';
|
|
2
2
|
import { createExtractSession } from '../create-extract-session.js';
|
|
3
|
-
import {
|
|
3
|
+
import { createExtractRuntime } from '../create-extract-runtime.js';
|
|
4
|
+
import { assertExtractRuntimeMatchesManifest } from '../extract-store.js';
|
|
4
5
|
import { DEFAULT_MAX_TOKENS } from './constants.js';
|
|
5
|
-
import { readManifest } from './manifest.js';
|
|
6
|
+
import { getManifestProvider, readManifest } from './manifest.js';
|
|
6
7
|
import { renderExtractPrompt } from './render-prompt.js';
|
|
7
8
|
import { resolveStoreDir } from './store.js';
|
|
8
9
|
export async function runExtractCommand(options) {
|
|
@@ -22,15 +23,20 @@ export async function runExtractCommand(options) {
|
|
|
22
23
|
if (options.dryRun) {
|
|
23
24
|
return renderExtractPrompt({ materials: manifest.materials }, request);
|
|
24
25
|
}
|
|
25
|
-
const runtime = await
|
|
26
|
+
const runtime = await createExtractRuntime({
|
|
26
27
|
model: manifest.model,
|
|
28
|
+
provider: getManifestProvider(manifest),
|
|
27
29
|
cacheDir: storeDir,
|
|
30
|
+
...(manifest.backend ? { backend: manifest.backend } : {}),
|
|
31
|
+
...(manifest.maxImageSize !== undefined ? { maxImageSize: manifest.maxImageSize } : {}),
|
|
28
32
|
});
|
|
29
33
|
try {
|
|
34
|
+
assertExtractRuntimeMatchesManifest(runtime, manifest);
|
|
30
35
|
const session = createExtractSession({
|
|
31
36
|
driver: runtime.driver,
|
|
32
37
|
cacheController: runtime.cacheController,
|
|
33
38
|
model: runtime.model,
|
|
39
|
+
maxImageSize: runtime.maxImageSize,
|
|
34
40
|
corpus: { materials: manifest.materials },
|
|
35
41
|
});
|
|
36
42
|
const result = await session.extract(request);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-command.js","sourceRoot":"","sources":["../../src/cli/extract-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"extract-command.js","sourceRoot":"","sources":["../../src/cli/extract-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,mCAAmC,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAW7C,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAA8B;IACpE,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAElE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IAE9C,MAAM,OAAO,GAAG;QACd,GAAG,EAAE,OAAO,CAAC,KAAK;QAClB,OAAO,EAAE;YACP,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;YAClD,WAAW,EAAE,CAAU;SACxB;KACF,CAAC;IAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,mBAAmB,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC;QACzC,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,QAAQ,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QACvC,QAAQ,EAAE,QAAQ;QAClB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,GAAG,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxF,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,mCAAmC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,oBAAoB,CAAC;YACnC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,MAAM,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE;SAC1C,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7C,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"list-command.d.ts","sourceRoot":"","sources":["../../src/cli/list-command.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;CACrB;AAsBD,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"list-command.d.ts","sourceRoot":"","sources":["../../src/cli/list-command.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,KAAK,GAAG,SAAS,CAAC;IAC5B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;CACrB;AAsBD,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAwB1E;AAoBD,wBAAsB,cAAc,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAOjF"}
|