akm-cli 0.9.15 → 0.9.16-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +144 -0
- package/dist/assets/tasks/core/index-refresh.yml +1 -1
- package/dist/cli/retired-commands.js +2 -0
- package/dist/cli/unknown-flags.js +36 -3
- package/dist/commands/improve/collapse-detector.js +2 -2
- package/dist/commands/improve/consolidate.js +6 -4
- package/dist/commands/improve/improve-cli.js +1 -1
- package/dist/commands/proposal/repository.js +12 -3
- package/dist/commands/read/curate.js +34 -44
- package/dist/commands/read/search.js +50 -2
- package/dist/commands/sources/index-status.js +99 -0
- package/dist/commands/sources/info.js +8 -8
- package/dist/commands/sources/installed-stashes.js +33 -12
- package/dist/commands/sources/source-add.js +21 -6
- package/dist/commands/sources/stash-cli.js +119 -111
- package/dist/core/adapter/adapters/akm-adapter.js +35 -3
- package/dist/core/adapter/adapters/akm-metadata.js +11 -1
- package/dist/core/asset/asset-placement.js +35 -0
- package/dist/core/config/schema/embedding.js +7 -30
- package/dist/core/config/schema/search.js +11 -9
- package/dist/core/errors.js +5 -2
- package/dist/core/hash.js +18 -0
- package/dist/core/maintenance-barrier.js +8 -6
- package/dist/core/paths.js +0 -11
- package/dist/core/run-lock.js +5 -2
- package/dist/core/state/migrations.js +26 -1
- package/dist/core/state-db.js +63 -27
- package/dist/indexer/drain.js +306 -0
- package/dist/indexer/embedding-identity.js +20 -0
- package/dist/indexer/enrich.js +260 -0
- package/dist/indexer/ensure-index.js +5 -0
- package/dist/indexer/index-written-assets.js +133 -171
- package/dist/indexer/indexer.js +458 -1621
- package/dist/indexer/lookup/adapter-concept-owner.js +19 -5
- package/dist/indexer/passes/metadata.js +18 -1
- package/dist/indexer/reconcile.js +890 -0
- package/dist/indexer/scan/drain-dir.js +27 -70
- package/dist/indexer/scan/parse-file.js +66 -0
- package/dist/indexer/search/db-search.js +373 -89
- package/dist/indexer/search/ranking-contributors.js +21 -16
- package/dist/indexer/search/ranking.js +135 -57
- package/dist/indexer/units/unit.js +159 -0
- package/dist/llm/client.js +10 -1
- package/dist/llm/embedder.js +10 -3
- package/dist/llm/embedders/provider-limits.js +288 -0
- package/dist/llm/embedders/remote.js +133 -104
- package/dist/llm/feature-gate.js +4 -2
- package/dist/llm/rerank-client.js +3 -3
- package/dist/output/shapes/passthrough.js +1 -0
- package/dist/output/text/command-format.js +19 -13
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/index.js +5 -2
- package/dist/scripts/akm-migrate-node.js +1141 -1237
- package/dist/scripts/akm-migrate.js +1141 -1237
- package/dist/setup/semantic-assets.js +2 -2
- package/dist/setup/steps/connection.js +3 -2
- package/dist/storage/repositories/files-repository.js +181 -0
- package/dist/storage/repositories/index-connection.js +1 -3
- package/dist/storage/repositories/index-entries-repository.js +77 -68
- package/dist/storage/repositories/index-entry-schema.js +16 -25
- package/dist/storage/repositories/index-fts-repository.js +29 -263
- package/dist/storage/repositories/index-meta-repository.js +0 -29
- package/dist/storage/repositories/index-schema.js +115 -122
- package/dist/storage/repositories/index-utility-repository.js +1 -1
- package/dist/storage/repositories/index-vec-repository.js +21 -334
- package/dist/storage/repositories/units-repository.js +510 -0
- package/docs/migration/release-notes/0.9.15.md +34 -36
- package/docs/migration/release-notes/0.9.16.md +110 -0
- package/docs/migration/release-notes/README.md +5 -0
- package/docs/reference/cli.md +93 -87
- package/docs/reference/configuration.md +128 -89
- package/docs/reference/data-and-telemetry.md +2 -1
- package/package.json +1 -1
- package/schemas/akm-config.json +2 -58
- package/dist/indexer/index-db-contention.js +0 -56
- package/dist/indexer/index-rebuild-lock.js +0 -73
- package/dist/indexer/materialize-embeddings.js +0 -771
- package/dist/indexer/passes/dir-staleness.js +0 -161
- package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
|
@@ -357,11 +357,17 @@ bundled.
|
|
|
357
357
|
## Indexing
|
|
358
358
|
|
|
359
359
|
AKM-native Markdown contributes a normalized body projection to the
|
|
360
|
-
lowest-weight `content` search field. The projection
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
360
|
+
lowest-weight `content` search field. The projection removes frontmatter,
|
|
361
|
+
comments, fenced code, and link destinations, and is never produced for
|
|
362
|
+
secret, env, session, or session-checkpoint assets.
|
|
363
|
+
|
|
364
|
+
Embedding input is not capped or truncated at all (index redesign): each
|
|
365
|
+
entry's structured fields (name/description/tags/hints) become one "card"
|
|
366
|
+
unit and each Markdown fragment becomes one "fragment" unit
|
|
367
|
+
(`src/indexer/units/unit.ts`), and a unit whose text would still exceed the
|
|
368
|
+
embedding provider's own probed window is split into ordinal sub-units that
|
|
369
|
+
share its fragment id — never truncated. See [Semantic
|
|
370
|
+
search](#semantic-search) for how that window is probed.
|
|
365
371
|
|
|
366
372
|
## Semantic search
|
|
367
373
|
|
|
@@ -396,76 +402,80 @@ unless a remote `embedding` config is provided.
|
|
|
396
402
|
`akm improve`'s memory-inference/consolidate passes when they call an
|
|
397
403
|
embedding model: `provider`, `endpoint`, `model`, `apiKey` (symbolic
|
|
398
404
|
reference, same rules as engine `apiKey`), `dimension`, `localModel`,
|
|
399
|
-
`
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
405
|
+
`timeoutMs`, `concurrency`, and `ollamaOptions.num_ctx`.
|
|
406
|
+
|
|
407
|
+
Request PACKING — how many documents land in one HTTP request, and the
|
|
408
|
+
token budget that bounds it — is no longer config at all. `akm index`
|
|
409
|
+
probes the embedding endpoint itself (llama.cpp's `GET /props`, Ollama's
|
|
410
|
+
`POST /api/show`) for its real context window and in-flight slot count
|
|
411
|
+
before packing any request, calibrating its chars-per-token estimate
|
|
412
|
+
against the provider's own tokenizer (llama.cpp's `/tokenize`) where it
|
|
413
|
+
offers one; an endpoint that answers neither probe (an OpenAI-compatible server, a gateway) gets a
|
|
414
|
+
conservative built-in default. This replaced four retired keys —
|
|
415
|
+
`maxInputTokens`, `maxTokens`, `batchSize`, `contextLength` — see
|
|
416
|
+
[Retired Configuration](#retired-configuration).
|
|
417
|
+
|
|
418
|
+
The knobs that remain, both optional (defaults apply when unset), for a
|
|
419
|
+
remote endpoint (`src/llm/embedders/remote.ts`):
|
|
404
420
|
|
|
405
421
|
| Key | Default | Bounds |
|
|
406
422
|
| --- | --- | --- |
|
|
407
|
-
| `embedding.maxInputTokens` | `512` | Per-DOCUMENT cap, applied before batching (#956). A document's embedded text is truncated to its head (unicode-safe) at this many estimated tokens instead of ever being skipped for size alone — a document is skipped only when its truncated head is empty. |
|
|
408
|
-
| `embedding.maxTokens` | `6000` (`DEFAULT_TOKEN_BUDGET`) | Per-REQUEST token budget: how many (already-capped) documents' estimated tokens fit in one HTTP request. With the 512-token default document cap, a request carries about 11 documents by default. Lowered from 8000 to 6000 (#954): the 4-chars-per-token estimator undercounts dense technical text by 7-55%, so 8000 regularly overshot an 8192-token endpoint's real context window. |
|
|
409
|
-
| `embedding.batchSize` | `100` | Per-REQUEST document-COUNT safety cap, independent of the token budget — guards against many tiny documents packing an oversized request. |
|
|
410
|
-
| `embedding.contextLength` | unset | Ollama's `num_ctx` ONLY, forwarded verbatim as `options.num_ctx` on the native `/api/embed` request. Does **not** feed the request token budget above (#956) — the two used to share this one field, so setting it for the server's context window silently changed request batching too. |
|
|
411
423
|
| `embedding.timeoutMs` | `120000` (120s) | Per-request wall timeout — see below. |
|
|
412
|
-
| `embedding.concurrency` | `1` loopback / `2` remote | In-flight request window — see below. |
|
|
413
|
-
|
|
414
|
-
**
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
request
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
counted, so the original 8.5k-12.4k-token documents that overflowed the
|
|
445
|
-
8192-token endpoint can never reach the request budget in the first place —
|
|
446
|
-
independent of whatever `maxTokens` or `contextLength` are set to.
|
|
424
|
+
| `embedding.concurrency` | `1` loopback / `2` remote, or the provider's own probed slot count | In-flight request window — see below. |
|
|
425
|
+
|
|
426
|
+
**What used to fix the field's 8k-context overflow.** A 0.9.15-beta field
|
|
427
|
+
report described documents estimated under the request budget that still
|
|
428
|
+
tokenized to 8.5k-12.4k real tokens against an 8192-token endpoint, because
|
|
429
|
+
the 4-chars-per-token estimator undercounts dense technical text.
|
|
430
|
+
0.9.15-beta answered it with three of the four now-retired keys; none of
|
|
431
|
+
them does anything in this release (see [Retired
|
|
432
|
+
Configuration](#retired-configuration)), but the shape of the old fix is
|
|
433
|
+
worth knowing because the probed-window replacement above closes the same
|
|
434
|
+
gap structurally instead of by convention:
|
|
435
|
+
|
|
436
|
+
- `embedding.maxInputTokens: 512` used to be a per-DOCUMENT cap, applied
|
|
437
|
+
before batching. This was the actual fix for the original overflow: no
|
|
438
|
+
single document could contribute more than 512 estimated tokens to a
|
|
439
|
+
request, no matter how `maxTokens` or `contextLength` were set.
|
|
440
|
+
- `embedding.maxTokens: 6000` used to be the per-REQUEST budget: how many
|
|
441
|
+
already-capped documents' estimated tokens fit in one HTTP request. A
|
|
442
|
+
request-level budget alone could not stop one oversized document from
|
|
443
|
+
overflowing a request — only the per-document cap above did that.
|
|
444
|
+
- `embedding.contextLength: 8192` fed only Ollama's `num_ctx`, forwarded
|
|
445
|
+
verbatim on a native `/api/embed` request. It never bounded request or
|
|
446
|
+
document sizing, and had no effect at all against a non-Ollama endpoint.
|
|
447
|
+
|
|
448
|
+
None of the three is read any more. A field config still carrying
|
|
449
|
+
`contextLength: 8192` + `maxTokens: 8000` (the exact 0.9.15-beta values
|
|
450
|
+
from the original report) loads on this release without error and without
|
|
451
|
+
effect — the same "ignored, unvalidated, no warning" handling every retired
|
|
452
|
+
key gets (see below) — because `akm index` now probes the endpoint's real
|
|
453
|
+
context window itself and packs every request against that instead of
|
|
454
|
+
against a configured estimate, so the original overflow is unreachable
|
|
455
|
+
regardless of what either retired key is set to.
|
|
447
456
|
|
|
448
457
|
`embedding.timeoutMs` (positive integer, default `120000` — 120s) is the
|
|
449
|
-
budget for a request at the FULL token budget
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
458
|
+
budget for a request at the FULL (probed) token budget; a local model
|
|
459
|
+
server on a large, token-budget-bounded batch legitimately takes longer
|
|
460
|
+
than the prior fixed 30s cut off. A smaller request gets a proportionally
|
|
461
|
+
smaller timeout —
|
|
453
462
|
`clamp(timeoutMs × requestTokens / tokenBudget, 30000, timeoutMs)` — so a
|
|
454
463
|
dead endpoint is still detected in seconds on the common case of small
|
|
455
464
|
documents. Set `embedding.timeoutMs` lower to fail fast against a
|
|
456
465
|
known-fast endpoint, or higher for a slow local server on large batches.
|
|
457
466
|
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
467
|
+
The probed token budget is also a run-scoped adaptive starting point, not a
|
|
468
|
+
hard ceiling, but ONLY when the endpoint did not report a real window of
|
|
469
|
+
its own (an OpenAI-compatible server or gateway, the conservative built-in
|
|
470
|
+
default): on the FIRST rejection of an `akm index` run for exceeding the
|
|
471
|
+
endpoint's context window, akm shrinks the request budget to three quarters
|
|
472
|
+
of its current value for every request not yet sent, and prints one line
|
|
473
|
+
naming the new value. A window akm actually probed from the endpoint
|
|
474
|
+
(llama.cpp, Ollama) is treated as authoritative and is never second-guessed
|
|
475
|
+
this way — a rejection against it still recovers via the same
|
|
476
|
+
split-and-retry every rejection gets (below), just without lowering the
|
|
477
|
+
budget for the rest of the run. This never fires a second time in the same
|
|
478
|
+
run either way.
|
|
469
479
|
|
|
470
480
|
A request TIMEOUT (not a rejection for exceeding the context window) never
|
|
471
481
|
drops its batch immediately: field confirmation showed that once akm
|
|
@@ -485,19 +495,17 @@ requests and reports failure — batches already committed are kept; rerun
|
|
|
485
495
|
once (a remote endpoint only; the local transformer path is unaffected):
|
|
486
496
|
`1` for a loopback endpoint (`localhost`, `127.0.0.0/8`, etc. — a local
|
|
487
497
|
model server serves one inference at a time, and parallel requests thrash
|
|
488
|
-
it) and `2` for a remote one, unless
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
498
|
+
it) and `2` for a remote one, unless the provider's own probed slot count
|
|
499
|
+
(llama.cpp's `total_slots`) or `embedding.concurrency` (positive integer,
|
|
500
|
+
1-16, checked first) overrides it. This default holds for the overwhelming
|
|
501
|
+
majority of setups; set an explicit override only for an endpoint that
|
|
502
|
+
genuinely serves parallel requests — a local server started with a
|
|
503
|
+
multi-slot flag (llama.cpp's `--parallel N`, vLLM) — not to "speed up" an
|
|
504
|
+
ordinary single-slot model server, which the default already protects from
|
|
494
505
|
reload-thrash. Request SIZE remains the first throughput lever regardless:
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
documents land in one request — with the default 512-token
|
|
499
|
-
`embedding.maxInputTokens` document cap, that is about 11 documents,
|
|
500
|
-
taking about the same wall time as a single one against a healthy endpoint.
|
|
506
|
+
the probed token budget and document-count cap described above control how
|
|
507
|
+
many documents land in one request, taking about the same wall time as a
|
|
508
|
+
single one against a healthy endpoint.
|
|
501
509
|
|
|
502
510
|
## Search tuning
|
|
503
511
|
|
|
@@ -505,7 +513,6 @@ taking about the same wall time as a single one against a healthy endpoint.
|
|
|
505
513
|
|
|
506
514
|
| Key | Purpose |
|
|
507
515
|
| --- | --- |
|
|
508
|
-
| `search.minScore` | Drop results below this score |
|
|
509
516
|
| `search.defaultExcludeTypes` | Asset types excluded from results by default |
|
|
510
517
|
|
|
511
518
|
### Graph boost search tuning
|
|
@@ -514,22 +521,35 @@ taking about the same wall time as a single one against a healthy endpoint.
|
|
|
514
521
|
| --- | --- |
|
|
515
522
|
| `search.graphBoost.*` | Entity-graph relevance boost: `directBoostPerEntity`/`directBoostCap` (directly related entities), `hopBoostPerEntity`/`hopBoostCap` (multi-hop, capped at `maxHops` ≤ 3), `confidenceMode` (`blend`, the only supported value), `confidenceWeight` (0–1, default `0.2`) |
|
|
516
523
|
|
|
517
|
-
###
|
|
524
|
+
### Search rerank (#951)
|
|
525
|
+
|
|
526
|
+
An optional cross-encoder rerank pass over `akm search`'s already-ranked
|
|
527
|
+
LOCAL stash hits, via a standalone `/rerank`-style HTTP endpoint (NOT one of
|
|
528
|
+
the `engines.*` `"llm"`/`"agent"` kinds). Disabled by default; a
|
|
529
|
+
misconfigured endpoint, network failure, timeout, or malformed response
|
|
530
|
+
falls back to search's own ranking unchanged. Applied once to local hits
|
|
531
|
+
before `--from local`/`--from all` diverge, so both see the reranked order
|
|
532
|
+
and neither double-applies it; registry hits (`--from registry`, and the
|
|
533
|
+
registry half of `--from all`) are never reranked and never trigger the
|
|
534
|
+
endpoint — registry results staying separate from stash hits is a locked
|
|
535
|
+
contract (AGENTS.md).
|
|
518
536
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
curate
|
|
537
|
+
Reranking changes hit ORDER only. Each hit's `score` is left as the
|
|
538
|
+
retrieval score `akm search` already computed — see the score-vs-order note
|
|
539
|
+
in `docs/reference/cli.md`'s search section for why.
|
|
540
|
+
|
|
541
|
+
Moved here from `akm curate` in 0.9.16 (`search.curateRerank` →
|
|
542
|
+
`search.rerank`) — the pass was always meant for search, not curate; see
|
|
543
|
+
`docs/migration/release-notes/0.9.16.md`.
|
|
524
544
|
|
|
525
545
|
| Key | Purpose |
|
|
526
546
|
| --- | --- |
|
|
527
|
-
| `search.
|
|
528
|
-
| `search.
|
|
529
|
-
| `search.
|
|
530
|
-
| `search.
|
|
531
|
-
| `search.
|
|
532
|
-
| `search.
|
|
547
|
+
| `search.rerank.enabled` | Turn the rerank pass on (default `false`) |
|
|
548
|
+
| `search.rerank.endpoint` | Full URL of the reranker's rerank endpoint |
|
|
549
|
+
| `search.rerank.model` | Model name sent to the endpoint (optional) |
|
|
550
|
+
| `search.rerank.apiKey` | `$VAR`/`secret://<name>` credential reference (optional) |
|
|
551
|
+
| `search.rerank.timeoutMs` | Request timeout (default `10000`) |
|
|
552
|
+
| `search.rerank.topN` | How many of search's ranked LOCAL hits to send (default `8`, max `50`) |
|
|
533
553
|
|
|
534
554
|
## Feedback
|
|
535
555
|
|
|
@@ -804,3 +824,22 @@ profile identities.
|
|
|
804
824
|
`embedding.chunkSize` was never read by anything under `src/` (#954), so a
|
|
805
825
|
config that still sets it is simply ignored — it still loads, unvalidated
|
|
806
826
|
and without warning.
|
|
827
|
+
|
|
828
|
+
`search.minScore` was never read by anything under `src/` as of 0.9
|
|
829
|
+
(index-redesign B5c), so a config that still sets it is simply ignored — it
|
|
830
|
+
still loads, unvalidated and without warning. It used to tune a
|
|
831
|
+
semantic-only floor over the old entries_fts + entries_vec search path,
|
|
832
|
+
calibrated for that path's 0-1 cosine/BM25 scores. The single search path is
|
|
833
|
+
`units_fts` + `units_vec`, scoring lexical evidence by BM25 magnitude and
|
|
834
|
+
semantic evidence by cosine similarity on the same 0.7/0.3 split as before,
|
|
835
|
+
on the scale the ranking contributors expect — no separate floor is applied.
|
|
836
|
+
|
|
837
|
+
`embedding.maxInputTokens`, `embedding.maxTokens`, `embedding.batchSize`,
|
|
838
|
+
and `embedding.contextLength` are retired (index redesign): `akm index`
|
|
839
|
+
packs requests against the embedding provider's own probed context window
|
|
840
|
+
and slot count instead (see [Semantic search](#semantic-search)). A config
|
|
841
|
+
that still sets any of them is simply ignored — it still loads, unvalidated
|
|
842
|
+
and without warning, the same as `embedding.chunkSize` above.
|
|
843
|
+
`embedding.contextLength` specifically fed Ollama's `num_ctx`; that request
|
|
844
|
+
field is now sent automatically from the same probe, or set explicitly via
|
|
845
|
+
`embedding.ollamaOptions.num_ctx`.
|
|
@@ -208,7 +208,8 @@ the set of types the code actually emits at HEAD (verified against every
|
|
|
208
208
|
| `events_purged` | Old events deleted by improve maintenance (90-day default retention) | `purgedCount`, `retentionDays` |
|
|
209
209
|
| `improve_runs_purged` | Old `improve_runs` rows deleted by improve maintenance (same retention window as events) | `purgedCount`, `retentionDays` |
|
|
210
210
|
| `improve_cycle_metrics_purged` | Old `improve_cycle_metrics` rows (365-day retention) deleted by improve maintenance | `purgedCount`, `retentionDays` |
|
|
211
|
-
| `task_logs_purged` | Old
|
|
211
|
+
| `task_logs_purged` | Old task-log rows (log lines in `logs.db`) deleted by improve maintenance | `purgedCount`, `retentionDays` |
|
|
212
|
+
| `task_log_files_purged` | Old per-run flat task log files (the transitional `<taskId>/<timestamp>.log` tail files under the task log dir) deleted by improve maintenance | `purgedCount`, `retentionDays` |
|
|
212
213
|
|
|
213
214
|
*Workflows*
|
|
214
215
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "akm-cli",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.16-alpha.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cursor.",
|
|
6
6
|
"keywords": [
|
package/schemas/akm-config.json
CHANGED
|
@@ -211,22 +211,6 @@
|
|
|
211
211
|
"type": "string",
|
|
212
212
|
"minLength": 1
|
|
213
213
|
},
|
|
214
|
-
"maxInputTokens": {
|
|
215
|
-
"type": "integer",
|
|
216
|
-
"exclusiveMinimum": 0
|
|
217
|
-
},
|
|
218
|
-
"maxTokens": {
|
|
219
|
-
"type": "integer",
|
|
220
|
-
"exclusiveMinimum": 0
|
|
221
|
-
},
|
|
222
|
-
"batchSize": {
|
|
223
|
-
"type": "integer",
|
|
224
|
-
"exclusiveMinimum": 0
|
|
225
|
-
},
|
|
226
|
-
"contextLength": {
|
|
227
|
-
"type": "integer",
|
|
228
|
-
"exclusiveMinimum": 0
|
|
229
|
-
},
|
|
230
214
|
"ollamaOptions": {
|
|
231
215
|
"type": "object",
|
|
232
216
|
"properties": {
|
|
@@ -546,10 +530,6 @@
|
|
|
546
530
|
"search": {
|
|
547
531
|
"type": "object",
|
|
548
532
|
"properties": {
|
|
549
|
-
"minScore": {
|
|
550
|
-
"type": "number",
|
|
551
|
-
"minimum": 0
|
|
552
|
-
},
|
|
553
533
|
"defaultExcludeTypes": {
|
|
554
534
|
"type": "array",
|
|
555
535
|
"items": {
|
|
@@ -597,7 +577,7 @@
|
|
|
597
577
|
},
|
|
598
578
|
"additionalProperties": true
|
|
599
579
|
},
|
|
600
|
-
"
|
|
580
|
+
"rerank": {
|
|
601
581
|
"type": "object",
|
|
602
582
|
"properties": {
|
|
603
583
|
"enabled": {
|
|
@@ -1939,22 +1919,6 @@
|
|
|
1939
1919
|
"type": "string",
|
|
1940
1920
|
"minLength": 1
|
|
1941
1921
|
},
|
|
1942
|
-
"maxInputTokens": {
|
|
1943
|
-
"type": "integer",
|
|
1944
|
-
"exclusiveMinimum": 0
|
|
1945
|
-
},
|
|
1946
|
-
"maxTokens": {
|
|
1947
|
-
"type": "integer",
|
|
1948
|
-
"exclusiveMinimum": 0
|
|
1949
|
-
},
|
|
1950
|
-
"batchSize": {
|
|
1951
|
-
"type": "integer",
|
|
1952
|
-
"exclusiveMinimum": 0
|
|
1953
|
-
},
|
|
1954
|
-
"contextLength": {
|
|
1955
|
-
"type": "integer",
|
|
1956
|
-
"exclusiveMinimum": 0
|
|
1957
|
-
},
|
|
1958
1922
|
"ollamaOptions": {
|
|
1959
1923
|
"type": "object",
|
|
1960
1924
|
"properties": {
|
|
@@ -2274,10 +2238,6 @@
|
|
|
2274
2238
|
"search": {
|
|
2275
2239
|
"type": "object",
|
|
2276
2240
|
"properties": {
|
|
2277
|
-
"minScore": {
|
|
2278
|
-
"type": "number",
|
|
2279
|
-
"minimum": 0
|
|
2280
|
-
},
|
|
2281
2241
|
"defaultExcludeTypes": {
|
|
2282
2242
|
"type": "array",
|
|
2283
2243
|
"items": {
|
|
@@ -2325,7 +2285,7 @@
|
|
|
2325
2285
|
},
|
|
2326
2286
|
"additionalProperties": true
|
|
2327
2287
|
},
|
|
2328
|
-
"
|
|
2288
|
+
"rerank": {
|
|
2329
2289
|
"type": "object",
|
|
2330
2290
|
"properties": {
|
|
2331
2291
|
"enabled": {
|
|
@@ -3545,22 +3505,6 @@
|
|
|
3545
3505
|
"type": "string",
|
|
3546
3506
|
"minLength": 1
|
|
3547
3507
|
},
|
|
3548
|
-
"maxInputTokens": {
|
|
3549
|
-
"type": "integer",
|
|
3550
|
-
"exclusiveMinimum": 0
|
|
3551
|
-
},
|
|
3552
|
-
"maxTokens": {
|
|
3553
|
-
"type": "integer",
|
|
3554
|
-
"exclusiveMinimum": 0
|
|
3555
|
-
},
|
|
3556
|
-
"batchSize": {
|
|
3557
|
-
"type": "integer",
|
|
3558
|
-
"exclusiveMinimum": 0
|
|
3559
|
-
},
|
|
3560
|
-
"contextLength": {
|
|
3561
|
-
"type": "integer",
|
|
3562
|
-
"exclusiveMinimum": 0
|
|
3563
|
-
},
|
|
3564
3508
|
"ollamaOptions": {
|
|
3565
3509
|
"type": "object",
|
|
3566
3510
|
"properties": {
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
-
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
-
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
/**
|
|
5
|
-
* Shared index.db contention reclassification (field follow-up to #956).
|
|
6
|
-
*
|
|
7
|
-
* Extracted out of `indexer.ts` so both `akmIndex`'s outer catch AND
|
|
8
|
-
* `generateEmbeddingsForDb`'s own catch (`materialize-embeddings.ts`) can
|
|
9
|
-
* reuse the ONE classifier instead of each building a raw
|
|
10
|
-
* `Semantic search verification failed: <driver message>` string. Living in
|
|
11
|
-
* its own module (rather than one importing the other) avoids the import
|
|
12
|
-
* cycle `indexer.ts` <-> `materialize-embeddings.ts` would otherwise form.
|
|
13
|
-
*/
|
|
14
|
-
import { AkmError, TransientError } from "../core/errors.js";
|
|
15
|
-
import { probeLock } from "../core/file-lock.js";
|
|
16
|
-
import { formatLockHolderPid } from "../core/run-lock.js";
|
|
17
|
-
import { isSqliteContentionError } from "../core/state-db.js";
|
|
18
|
-
import { indexRebuildLockPath } from "./index-rebuild-lock.js";
|
|
19
|
-
/**
|
|
20
|
-
* Read-only description of the rebuild lock's current holder, appended to a
|
|
21
|
-
* reclassified index.db contention message when known (field follow-up to
|
|
22
|
-
* #956). `probeLock` only inspects the sentinel — it never acquires or
|
|
23
|
-
* mutates it — so this is safe to call from inside an error path.
|
|
24
|
-
*/
|
|
25
|
-
function describeIndexRebuildLockHolder() {
|
|
26
|
-
const probe = probeLock(indexRebuildLockPath());
|
|
27
|
-
if (probe.state !== "held")
|
|
28
|
-
return "";
|
|
29
|
-
return ` The rebuild lock is currently held by pid ${formatLockHolderPid({
|
|
30
|
-
pid: probe.holderPid,
|
|
31
|
-
launcherPid: probe.launcherPid ?? null,
|
|
32
|
-
})}.`;
|
|
33
|
-
}
|
|
34
|
-
/**
|
|
35
|
-
* Reclassify a contention-shaped error escaping the walk, index, or
|
|
36
|
-
* embedding phase into a retryable-shortly `TransientError` (field
|
|
37
|
-
* follow-up to #956, dev-team field review 2026-09-10): a concurrent writer
|
|
38
|
-
* (another `akm index`, a source-update embedding pass, the per-command
|
|
39
|
-
* background reindex) can make index.db busy, and the raw SQLite driver
|
|
40
|
-
* error ("database is locked") used to escape as exit 70
|
|
41
|
-
* (internal/unclassified) instead of the "retry shortly" contract exit 75
|
|
42
|
-
* gives a scheduler to branch on — mirroring `STATE_DB_CONTENDED`'s
|
|
43
|
-
* precedent for state.db (`core/state-db.ts`). Reuses the ONE shared
|
|
44
|
-
* classifier, `isSqliteContentionError`, rather than a second one. An error
|
|
45
|
-
* that is already a classified akm error (e.g. a `STATE_DB_CONTENDED`
|
|
46
|
-
* TransientError from an inner state.db write) is never re-wrapped — only a
|
|
47
|
-
* raw, unclassified error matching the shared contention shape is
|
|
48
|
-
* reclassified. Every other error is rethrown unchanged.
|
|
49
|
-
*/
|
|
50
|
-
export function reclassifyIndexDbContention(error) {
|
|
51
|
-
if (error instanceof AkmError || !isSqliteContentionError(error))
|
|
52
|
-
return error;
|
|
53
|
-
const contended = new TransientError(`akm's index database is busy (another akm process is writing it); retry shortly.${describeIndexRebuildLockHolder()}`, "INDEX_DB_CONTENDED");
|
|
54
|
-
contended.cause = error;
|
|
55
|
-
return contended;
|
|
56
|
-
}
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
-
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
-
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
/**
|
|
5
|
-
* Opt-in, non-blocking rebuild lock for `akm index` (#956).
|
|
6
|
-
*
|
|
7
|
-
* #872 removed the blocking index-rebuild lease: the index is a regenerable
|
|
8
|
-
* cache, so a concurrent rebuild only wastes work rather than corrupts
|
|
9
|
-
* anything, and a live-but-wedged holder passed a PID-liveness check forever
|
|
10
|
-
* — only an age-based clock could ever free it, which is exactly the hazard
|
|
11
|
-
* #872 deleted. This module does not reinstate that lock. It adds a
|
|
12
|
-
* PID-liveness-only sentinel an explicit `akm index` run acquires and
|
|
13
|
-
* releases on exit purely so a *scheduled or opportunistic* run
|
|
14
|
-
* (`--skip-if-locked`) can step aside instead of piling up behind a rebuild
|
|
15
|
-
* already in progress. A human-typed `akm index` with no flag is never
|
|
16
|
-
* gated: it warns and proceeds exactly as it did before this lock existed.
|
|
17
|
-
*
|
|
18
|
-
* Built on the shared PID-liveness mechanics in `core/run-lock.ts` (the same
|
|
19
|
-
* ones `akm improve`'s whole-run lock uses) — see that module's doc for the
|
|
20
|
-
* no-stale-age-window rationale.
|
|
21
|
-
*/
|
|
22
|
-
import { releaseLock } from "../core/file-lock.js";
|
|
23
|
-
import { tryWithMaintenanceStartBarrier, withMaintenanceStartBarrier } from "../core/maintenance-barrier.js";
|
|
24
|
-
import { getIndexRebuildLockPath } from "../core/paths.js";
|
|
25
|
-
import { formatLockHolderPid, tryAcquireRunLock } from "../core/run-lock.js";
|
|
26
|
-
import { warn, warnVerbose } from "../core/warn.js";
|
|
27
|
-
export function indexRebuildLockPath() {
|
|
28
|
-
return getIndexRebuildLockPath();
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* Acquire the rebuild lock for the duration of one `akm index` run.
|
|
32
|
-
*
|
|
33
|
-
* - Free: always returns `"acquired"`.
|
|
34
|
-
* - Held, `skipIfLocked`: warns once (naming the holder) and returns
|
|
35
|
-
* `"skipped"` — the caller must not run `akmIndex()` at all.
|
|
36
|
-
* - Held, no flag: warns once and returns `"contended"` — the caller runs
|
|
37
|
-
* `akmIndex()` unlocked, exactly as every `akm index` did before #956.
|
|
38
|
-
*
|
|
39
|
-
* A dead holder's lease is reclaimed silently (verbose-only log line, never
|
|
40
|
-
* a user-facing warning) — the operator did nothing wrong and nothing here
|
|
41
|
-
* requires their attention.
|
|
42
|
-
*/
|
|
43
|
-
export function tryAcquireIndexRebuildLock(skipIfLocked) {
|
|
44
|
-
const lockPath = indexRebuildLockPath();
|
|
45
|
-
const acquire = () => tryAcquireRunLock(lockPath, {
|
|
46
|
-
label: "index rebuild",
|
|
47
|
-
onReclaimed: (info) => {
|
|
48
|
-
warnVerbose(`[index] reclaimed a rebuild lock left by pid ${info.holderPid ?? "unknown"} ` +
|
|
49
|
-
`(${info.reason}); that process is no longer running.`);
|
|
50
|
-
},
|
|
51
|
-
});
|
|
52
|
-
if (skipIfLocked) {
|
|
53
|
-
const result = tryWithMaintenanceStartBarrier(acquire);
|
|
54
|
-
if (!result) {
|
|
55
|
-
warn("[index] maintenance barrier held; skipping (--skip-if-locked)");
|
|
56
|
-
return { state: "skipped", holder: { pid: null, startedAt: null, launcherPid: null } };
|
|
57
|
-
}
|
|
58
|
-
if (result.state === "acquired")
|
|
59
|
-
return result;
|
|
60
|
-
warn(`[index] another index run holds the lock (PID ${formatLockHolderPid(result.holder)}, started ${result.holder.startedAt}); ` +
|
|
61
|
-
"skipping (--skip-if-locked)");
|
|
62
|
-
return { state: "skipped", holder: result.holder };
|
|
63
|
-
}
|
|
64
|
-
const result = withMaintenanceStartBarrier(acquire);
|
|
65
|
-
if (result.state === "acquired")
|
|
66
|
-
return result;
|
|
67
|
-
warn(`[index] another index run is active (pid ${formatLockHolderPid(result.holder)}, started ${result.holder.startedAt}); ` +
|
|
68
|
-
"this run will contend with it — pass --skip-if-locked for scheduled runs");
|
|
69
|
-
return { state: "contended", holder: result.holder };
|
|
70
|
-
}
|
|
71
|
-
export function releaseIndexRebuildLock(ownership) {
|
|
72
|
-
releaseLock(ownership);
|
|
73
|
-
}
|