akm-cli 0.9.15 → 0.9.16-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +144 -0
- package/dist/assets/tasks/core/index-refresh.yml +1 -1
- package/dist/cli/retired-commands.js +2 -0
- package/dist/cli/unknown-flags.js +36 -3
- package/dist/commands/improve/collapse-detector.js +2 -2
- package/dist/commands/improve/consolidate.js +6 -4
- package/dist/commands/improve/improve-cli.js +1 -1
- package/dist/commands/proposal/repository.js +12 -3
- package/dist/commands/read/curate.js +34 -44
- package/dist/commands/read/search.js +50 -2
- package/dist/commands/sources/index-status.js +99 -0
- package/dist/commands/sources/info.js +8 -8
- package/dist/commands/sources/installed-stashes.js +33 -12
- package/dist/commands/sources/source-add.js +21 -6
- package/dist/commands/sources/stash-cli.js +119 -111
- package/dist/core/adapter/adapters/akm-adapter.js +35 -3
- package/dist/core/adapter/adapters/akm-metadata.js +11 -1
- package/dist/core/asset/asset-placement.js +35 -0
- package/dist/core/config/schema/embedding.js +7 -30
- package/dist/core/config/schema/search.js +11 -9
- package/dist/core/errors.js +5 -2
- package/dist/core/hash.js +18 -0
- package/dist/core/maintenance-barrier.js +8 -6
- package/dist/core/paths.js +0 -11
- package/dist/core/run-lock.js +5 -2
- package/dist/core/state/migrations.js +26 -1
- package/dist/core/state-db.js +63 -27
- package/dist/indexer/drain.js +306 -0
- package/dist/indexer/embedding-identity.js +20 -0
- package/dist/indexer/enrich.js +260 -0
- package/dist/indexer/ensure-index.js +5 -0
- package/dist/indexer/index-written-assets.js +133 -171
- package/dist/indexer/indexer.js +458 -1621
- package/dist/indexer/lookup/adapter-concept-owner.js +19 -5
- package/dist/indexer/passes/metadata.js +18 -1
- package/dist/indexer/reconcile.js +890 -0
- package/dist/indexer/scan/drain-dir.js +27 -70
- package/dist/indexer/scan/parse-file.js +66 -0
- package/dist/indexer/search/db-search.js +373 -89
- package/dist/indexer/search/ranking-contributors.js +21 -16
- package/dist/indexer/search/ranking.js +135 -57
- package/dist/indexer/units/unit.js +159 -0
- package/dist/llm/client.js +10 -1
- package/dist/llm/embedder.js +10 -3
- package/dist/llm/embedders/provider-limits.js +288 -0
- package/dist/llm/embedders/remote.js +133 -104
- package/dist/llm/feature-gate.js +4 -2
- package/dist/llm/rerank-client.js +3 -3
- package/dist/output/shapes/passthrough.js +1 -0
- package/dist/output/text/command-format.js +19 -13
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/index.js +5 -2
- package/dist/scripts/akm-migrate-node.js +1141 -1237
- package/dist/scripts/akm-migrate.js +1141 -1237
- package/dist/setup/semantic-assets.js +2 -2
- package/dist/setup/steps/connection.js +3 -2
- package/dist/storage/repositories/files-repository.js +181 -0
- package/dist/storage/repositories/index-connection.js +1 -3
- package/dist/storage/repositories/index-entries-repository.js +77 -68
- package/dist/storage/repositories/index-entry-schema.js +16 -25
- package/dist/storage/repositories/index-fts-repository.js +29 -263
- package/dist/storage/repositories/index-meta-repository.js +0 -29
- package/dist/storage/repositories/index-schema.js +115 -122
- package/dist/storage/repositories/index-utility-repository.js +1 -1
- package/dist/storage/repositories/index-vec-repository.js +21 -334
- package/dist/storage/repositories/units-repository.js +510 -0
- package/docs/migration/release-notes/0.9.15.md +34 -36
- package/docs/migration/release-notes/0.9.16.md +110 -0
- package/docs/migration/release-notes/README.md +5 -0
- package/docs/reference/cli.md +93 -87
- package/docs/reference/configuration.md +128 -89
- package/docs/reference/data-and-telemetry.md +2 -1
- package/package.json +1 -1
- package/schemas/akm-config.json +2 -58
- package/dist/indexer/index-db-contention.js +0 -56
- package/dist/indexer/index-rebuild-lock.js +0 -73
- package/dist/indexer/materialize-embeddings.js +0 -771
- package/dist/indexer/passes/dir-staleness.js +0 -161
- package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
|
@@ -7044,7 +7044,8 @@ var init_errors = __esm(() => {
|
|
|
7044
7044
|
UNSAFE_STASH_DIR: "Choose a path inside your home directory (e.g. ~/akm) or another empty workspace. The bundle directory cannot be the filesystem root, your home directory itself, or a sensitive system path like /etc, /var, ~/.config, or ~/.ssh.",
|
|
7045
7045
|
UNKNOWN_IMPROVE_STRATEGY: "Pass one of the listed strategy names to `--strategy`, or define it under `improve.strategies`. Names are case-sensitive.",
|
|
7046
7046
|
EXECUTION_NOT_AUTHORIZED: "Change the selected tools or update the machine/user execution policy, then retry.",
|
|
7047
|
-
SECRET_REFERENCE_UNRESOLVED: "Check the secret exists (`akm secret list`) and the name after `secret://` matches, or run `akm secret set <name> <value>` to store it."
|
|
7047
|
+
SECRET_REFERENCE_UNRESOLVED: "Check the secret exists (`akm secret list`) and the name after `secret://` matches, or run `akm secret set <name> <value>` to store it.",
|
|
7048
|
+
EMBEDDING_VEC_UNAVAILABLE: "Install sqlite-vec for unit-level semantic search, or rely on lexical search until it is available."
|
|
7048
7049
|
};
|
|
7049
7050
|
COMPOSITION_INVALID_MULTI_JOB_HINT = "AKM workflows support exactly one job per source, with no needs: between jobs. Split the extra job(s) into " + "their own workflow file, and compose them with uses: workflows/<ref> instead.";
|
|
7050
7051
|
USAGE_HINTS = {
|
|
@@ -7071,8 +7072,8 @@ var init_errors = __esm(() => {
|
|
|
7071
7072
|
TRANSIENT_HINTS = {
|
|
7072
7073
|
RUN_LEASE_HELD: "Wait for the named engine invocation to finish or for the lease to expire, then retry. `akm workflow status <id>` shows the current lease.",
|
|
7073
7074
|
STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing.",
|
|
7074
|
-
INDEX_DB_CONTENDED: "Another akm process is writing index.db
|
|
7075
|
-
MAINTENANCE_BARRIER_BUSY: "Another akm process is registering a lock or lease right now. Retry shortly, or pass --skip-if-locked on scheduled
|
|
7075
|
+
INDEX_DB_CONTENDED: "Another akm process is writing index.db right now. Wait a few seconds and retry — index runs take no rebuild " + "lock, so this clears quickly; a scheduled run left alone will simply run again next time.",
|
|
7076
|
+
MAINTENANCE_BARRIER_BUSY: "Another akm process is registering a lock or lease right now. Retry shortly, or pass --skip-if-locked on " + "scheduled improve runs to skip gracefully instead — workflow run does not treat this code as skippable.",
|
|
7076
7077
|
IMPROVE_LOCK_HELD: "Another akm improve run holds the whole-run lock right now. Wait for it to finish and retry, or pass --skip-if-locked on scheduled runs."
|
|
7077
7078
|
};
|
|
7078
7079
|
NOT_FOUND_HINTS = {
|
|
@@ -7540,9 +7541,6 @@ function getDataDir(env = process.env, platform = process.platform) {
|
|
|
7540
7541
|
function getDbPath(env = process.env) {
|
|
7541
7542
|
return path3.join(getDataDir(env), "index.db");
|
|
7542
7543
|
}
|
|
7543
|
-
function getIndexRebuildLockPath() {
|
|
7544
|
-
return path3.join(getDataDir(), "index.rebuild.lock");
|
|
7545
|
-
}
|
|
7546
7544
|
function getMaintenanceBarrierPath() {
|
|
7547
7545
|
return path3.join(getDataDir(), "maintenance.barrier.lock");
|
|
7548
7546
|
}
|
|
@@ -8174,14 +8172,6 @@ var init_common = __esm(() => {
|
|
|
8174
8172
|
});
|
|
8175
8173
|
|
|
8176
8174
|
// src/core/recognition-util.ts
|
|
8177
|
-
function canonicalizeWorkflowName(name) {
|
|
8178
|
-
const lower = name.toLowerCase();
|
|
8179
|
-
for (const ext of WORKFLOW_EXTENSIONS) {
|
|
8180
|
-
if (lower.endsWith(ext))
|
|
8181
|
-
return name.slice(0, -ext.length);
|
|
8182
|
-
}
|
|
8183
|
-
return name;
|
|
8184
|
-
}
|
|
8185
8175
|
function isKnownType(type) {
|
|
8186
8176
|
return KNOWN_TYPES.includes(type);
|
|
8187
8177
|
}
|
|
@@ -8277,6 +8267,9 @@ function assetPathCandidatesForName(assetType, typeRoot, name) {
|
|
|
8277
8267
|
const namedForm = path7.join(typeRoot, base, "default.env");
|
|
8278
8268
|
return [...new Set([primary, dotForm, namedForm])];
|
|
8279
8269
|
}
|
|
8270
|
+
function assetPathCandidatesAreOrderedByPreference(assetType) {
|
|
8271
|
+
return assetType === "memory";
|
|
8272
|
+
}
|
|
8280
8273
|
var workflowSpec, markdownSpec, scriptSpec, BUILTIN_PLACEMENT_SPECS, PLACEMENT_SPECS;
|
|
8281
8274
|
var init_asset_placement = __esm(() => {
|
|
8282
8275
|
init_common();
|
|
@@ -11390,7 +11383,8 @@ function applyPostContributorFields(entry, file, canonicalName, dirPath) {
|
|
|
11390
11383
|
}
|
|
11391
11384
|
entry.tags = [...entry.tags ?? [], ...extractDirTagsFromName(canonicalName)];
|
|
11392
11385
|
entry.tags = normalizeTerms(entry.tags ?? []);
|
|
11393
|
-
|
|
11386
|
+
const aliasTagInput = entry.type === "memory" && canonicalName.toLowerCase().endsWith(".derived") ? entry.tags.filter((tag) => tag !== "derived") : entry.tags;
|
|
11387
|
+
entry.aliases = mergeAliases(entry.aliases, buildAliases(canonicalName, aliasTagInput));
|
|
11394
11388
|
entry.filename = path17.basename(file);
|
|
11395
11389
|
}
|
|
11396
11390
|
function buildMetadataSkipWarning(filePath, assetType, error) {
|
|
@@ -36378,7 +36372,7 @@ var require_libvips = __commonJS((exports, module) => {
|
|
|
36378
36372
|
SPDX-License-Identifier: Apache-2.0
|
|
36379
36373
|
*/
|
|
36380
36374
|
var { spawnSync: spawnSync6 } = __require("node:child_process");
|
|
36381
|
-
var { createHash:
|
|
36375
|
+
var { createHash: createHash10 } = __require("node:crypto");
|
|
36382
36376
|
var semverCoerce = require_coerce2();
|
|
36383
36377
|
var semverGreaterThanOrEqualTo = require_gte2();
|
|
36384
36378
|
var semverSatisfies = require_satisfies2();
|
|
@@ -36466,7 +36460,7 @@ var require_libvips = __commonJS((exports, module) => {
|
|
|
36466
36460
|
}
|
|
36467
36461
|
return false;
|
|
36468
36462
|
};
|
|
36469
|
-
var sha512 = (s) =>
|
|
36463
|
+
var sha512 = (s) => createHash10("sha512").update(s).digest("hex");
|
|
36470
36464
|
var yarnLocator = () => {
|
|
36471
36465
|
try {
|
|
36472
36466
|
const identHash = sha512(`imgsharp-libvips-${buildPlatformArch()}`);
|
|
@@ -43918,8 +43912,8 @@ function cos_sim(arr1, arr2) {
|
|
|
43918
43912
|
const dotProduct = dot(arr1, arr2);
|
|
43919
43913
|
const magnitudeA = magnitude(arr1);
|
|
43920
43914
|
const magnitudeB = magnitude(arr2);
|
|
43921
|
-
const
|
|
43922
|
-
return
|
|
43915
|
+
const cosineSimilarity = dotProduct / (magnitudeA * magnitudeB);
|
|
43916
|
+
return cosineSimilarity;
|
|
43923
43917
|
}
|
|
43924
43918
|
function magnitude(arr) {
|
|
43925
43919
|
return Math.sqrt(arr.reduce((acc, val) => acc + val * val, 0));
|
|
@@ -68858,7 +68852,7 @@ ${this.boa_token}${this.audio_token.repeat(this._compute_audio_num_tokens(audio_
|
|
|
68858
68852
|
});
|
|
68859
68853
|
|
|
68860
68854
|
// src/indexer/walk/file-context.ts
|
|
68861
|
-
import
|
|
68855
|
+
import fs49 from "node:fs";
|
|
68862
68856
|
import path61 from "node:path";
|
|
68863
68857
|
function buildFileContext(stashRoot, absPath) {
|
|
68864
68858
|
const relPath = toPosix(path61.relative(stashRoot, absPath));
|
|
@@ -68883,7 +68877,7 @@ function buildFileContext(stashRoot, absPath) {
|
|
|
68883
68877
|
stashRoot,
|
|
68884
68878
|
content() {
|
|
68885
68879
|
if (cachedContent === undefined) {
|
|
68886
|
-
cachedContent =
|
|
68880
|
+
cachedContent = fs49.readFileSync(absPath, "utf8");
|
|
68887
68881
|
}
|
|
68888
68882
|
return cachedContent;
|
|
68889
68883
|
},
|
|
@@ -68898,7 +68892,7 @@ function buildFileContext(stashRoot, absPath) {
|
|
|
68898
68892
|
},
|
|
68899
68893
|
stat() {
|
|
68900
68894
|
if (cachedStat === undefined) {
|
|
68901
|
-
cachedStat =
|
|
68895
|
+
cachedStat = fs49.statSync(absPath);
|
|
68902
68896
|
}
|
|
68903
68897
|
return cachedStat;
|
|
68904
68898
|
}
|
|
@@ -69696,6 +69690,7 @@ var PASSTHROUGH_COMMANDS = [
|
|
|
69696
69690
|
"improve-report",
|
|
69697
69691
|
"import",
|
|
69698
69692
|
"index",
|
|
69693
|
+
"index-status",
|
|
69699
69694
|
"info",
|
|
69700
69695
|
"lint",
|
|
69701
69696
|
"list",
|
|
@@ -70640,7 +70635,7 @@ Config saved to ${r.configPath}`;
|
|
|
70640
70635
|
}
|
|
70641
70636
|
function formatIndexPlain(r) {
|
|
70642
70637
|
const indexResult = r;
|
|
70643
|
-
let out = `Indexed ${indexResult.totalEntries ?? 0} entries from ${indexResult.
|
|
70638
|
+
let out = `Indexed ${indexResult.totalEntries ?? 0} entries from ${indexResult.sourcesScanned ?? 0} source${indexResult.sourcesScanned === 1 ? "" : "s"} (mode: ${indexResult.mode ?? "unknown"})`;
|
|
70644
70639
|
const warnings = indexResult.warnings;
|
|
70645
70640
|
if (Array.isArray(warnings) && warnings.length > 0) {
|
|
70646
70641
|
out += `
|
|
@@ -70649,13 +70644,6 @@ Warnings (${warnings.length}):`;
|
|
|
70649
70644
|
out += `
|
|
70650
70645
|
- ${String(message)}`;
|
|
70651
70646
|
}
|
|
70652
|
-
const notices = Array.isArray(indexResult.notices) ? indexResult.notices : [];
|
|
70653
|
-
for (const notice of notices) {
|
|
70654
|
-
const severity = notice.severity === "info" ? "info" : "warning";
|
|
70655
|
-
const field = typeof notice.field === "string" ? ` field=${notice.field}` : "";
|
|
70656
|
-
out += `
|
|
70657
|
-
notice[${severity}] ${notice.code} adapter=${notice.adapter}${field}` + (notice.message ? `: ${notice.message}` : "");
|
|
70658
|
-
}
|
|
70659
70647
|
const verification = indexResult.verification;
|
|
70660
70648
|
if (verification?.ok === false && verification.message) {
|
|
70661
70649
|
out += `
|
|
@@ -70664,10 +70652,26 @@ Verification: ${String(verification.message)}`;
|
|
|
70664
70652
|
const timing = indexResult.timing;
|
|
70665
70653
|
if (timing) {
|
|
70666
70654
|
out += `
|
|
70667
|
-
Timing: total ${timing.totalMs}ms` + `, preflight ${timing.preflightMs}ms` + `,
|
|
70655
|
+
Timing: total ${timing.totalMs}ms` + `, preflight ${timing.preflightMs}ms` + `, source cache ${timing.sourceCacheMs}ms` + `, reconcile ${timing.reconcileMs}ms` + `, embeddings ${timing.embedMs}ms` + `, finalize ${timing.finalizeMs}ms` + `, end-to-end ${timing.endToEndMs}ms`;
|
|
70668
70656
|
}
|
|
70669
70657
|
return out;
|
|
70670
70658
|
}
|
|
70659
|
+
function formatIndexStatusPlain(r) {
|
|
70660
|
+
const units = r.units ?? {};
|
|
70661
|
+
const lines = [
|
|
70662
|
+
`Index: ${String(r.indexPath ?? "unknown")}`,
|
|
70663
|
+
`Files: ${Number(r.files ?? 0)}`,
|
|
70664
|
+
`Entries: ${Number(r.entries ?? 0)}`,
|
|
70665
|
+
`Units: ${Number(units.total ?? 0)} total, ${Number(units.withVector ?? 0)} with a vector, ${Number(units.pending ?? 0)} pending`,
|
|
70666
|
+
`Active identity: ${typeof r.activeIdentity === "string" ? r.activeIdentity : "none yet"}`,
|
|
70667
|
+
`Last reconcile: ${typeof r.lastReconcileAt === "string" ? r.lastReconcileAt : "never"}`,
|
|
70668
|
+
`Built at: ${typeof r.builtAt === "string" ? r.builtAt : "never"}`
|
|
70669
|
+
];
|
|
70670
|
+
if (typeof r.unreadable === "string")
|
|
70671
|
+
lines.push(`Unreadable: ${r.unreadable}`);
|
|
70672
|
+
return lines.join(`
|
|
70673
|
+
`);
|
|
70674
|
+
}
|
|
70671
70675
|
function formatListPlain(r) {
|
|
70672
70676
|
const sources = Array.isArray(r.sources) ? r.sources : [];
|
|
70673
70677
|
if (sources.length === 0)
|
|
@@ -75920,10 +75924,6 @@ var EmbeddingConnectionConfigSchema = exports_external.object({
|
|
|
75920
75924
|
apiKey: symbolicOrWarnApiKey("embedding.apiKey").optional(),
|
|
75921
75925
|
dimension: positiveInt.max(4096).optional(),
|
|
75922
75926
|
localModel: exports_external.string().min(1).optional(),
|
|
75923
|
-
maxInputTokens: positiveInt.optional(),
|
|
75924
|
-
maxTokens: positiveInt.optional(),
|
|
75925
|
-
batchSize: positiveInt.optional(),
|
|
75926
|
-
contextLength: positiveInt.optional(),
|
|
75927
75927
|
ollamaOptions: EmbeddingOllamaOptionsSchema.optional(),
|
|
75928
75928
|
timeoutMs: positiveInt.optional(),
|
|
75929
75929
|
concurrency: positiveInt.max(16).optional()
|
|
@@ -76379,19 +76379,18 @@ var SearchGraphBoostSchema = exports_external.object({
|
|
|
76379
76379
|
confidenceMode: exports_external.enum(["blend"]).default("blend").optional(),
|
|
76380
76380
|
confidenceWeight: exports_external.number().finite().min(0).max(1).default(0.2).optional()
|
|
76381
76381
|
}).passthrough();
|
|
76382
|
-
var
|
|
76382
|
+
var SearchRerankConfigSchema = exports_external.object({
|
|
76383
76383
|
enabled: exports_external.boolean().optional(),
|
|
76384
76384
|
endpoint: httpUrl.optional(),
|
|
76385
76385
|
model: nonEmptyString.optional(),
|
|
76386
|
-
apiKey: symbolicOrWarnApiKey("search.
|
|
76386
|
+
apiKey: symbolicOrWarnApiKey("search.rerank.apiKey").optional(),
|
|
76387
76387
|
timeoutMs: positiveInt.optional(),
|
|
76388
76388
|
topN: positiveInt.max(50).optional()
|
|
76389
76389
|
}).passthrough();
|
|
76390
76390
|
var SearchConfigSchema = exports_external.object({
|
|
76391
|
-
minScore: nonNegativeNumber.optional(),
|
|
76392
76391
|
defaultExcludeTypes: exports_external.array(nonEmptyString).optional(),
|
|
76393
76392
|
graphBoost: SearchGraphBoostSchema.optional(),
|
|
76394
|
-
|
|
76393
|
+
rerank: SearchRerankConfigSchema.optional()
|
|
76395
76394
|
}).passthrough();
|
|
76396
76395
|
|
|
76397
76396
|
// src/core/config/schema/setup.ts
|
|
@@ -83567,7 +83566,7 @@ function foldRecognizedMetadata(rendererName, file) {
|
|
|
83567
83566
|
const fm = parseFrontmatter(file.content()).data;
|
|
83568
83567
|
applyFrontmatterDescriptionAndTags(fm, out);
|
|
83569
83568
|
const hints = new Set;
|
|
83570
|
-
const source = nonEmptyString2(fm.source);
|
|
83569
|
+
const source = fm.inferred === true ? undefined : nonEmptyString2(fm.source);
|
|
83571
83570
|
if (source)
|
|
83572
83571
|
hints.add(source);
|
|
83573
83572
|
const fmObservedAt = nonEmptyString2(fm.observed_at);
|
|
@@ -84016,9 +84015,23 @@ var akmAdapter = {
|
|
|
84016
84015
|
return [];
|
|
84017
84016
|
const canonical = assetPathCandidatesForName(type, path24.join(c.root, head), rest);
|
|
84018
84017
|
const loose = assetPathCandidatesForName(type, c.root, rest);
|
|
84019
|
-
|
|
84018
|
+
if (!assetPathCandidatesAreOrderedByPreference(type)) {
|
|
84019
|
+
return [...new Set([...canonical, ...loose])].map((candidatePath) => ({
|
|
84020
|
+
path: candidatePath,
|
|
84021
|
+
conceptId: posix
|
|
84022
|
+
}));
|
|
84023
|
+
}
|
|
84024
|
+
const priorityByPath = new Map;
|
|
84025
|
+
for (const list of [canonical, loose]) {
|
|
84026
|
+
list.forEach((candidatePath, rank) => {
|
|
84027
|
+
if (!priorityByPath.has(candidatePath))
|
|
84028
|
+
priorityByPath.set(candidatePath, rank);
|
|
84029
|
+
});
|
|
84030
|
+
}
|
|
84031
|
+
return [...priorityByPath.keys()].map((candidatePath) => ({
|
|
84020
84032
|
path: candidatePath,
|
|
84021
|
-
conceptId: posix
|
|
84033
|
+
conceptId: posix,
|
|
84034
|
+
priority: priorityByPath.get(candidatePath)
|
|
84022
84035
|
}));
|
|
84023
84036
|
},
|
|
84024
84037
|
placeNew(c, conceptId) {
|
|
@@ -93484,6 +93497,7 @@ import { AsyncLocalStorage as AsyncLocalStorage3 } from "node:async_hooks";
|
|
|
93484
93497
|
var attributionStorage = new AsyncLocalStorage3;
|
|
93485
93498
|
// src/llm/client.ts
|
|
93486
93499
|
var jsonSchemaUnsupportedConnections = new Set;
|
|
93500
|
+
var HEALTH_PROBE_TIMEOUT_MS = 3000;
|
|
93487
93501
|
|
|
93488
93502
|
// src/integrations/agent/model-map.ts
|
|
93489
93503
|
init_common();
|
|
@@ -93675,7 +93689,10 @@ var improveReportFormatters = [
|
|
|
93675
93689
|
];
|
|
93676
93690
|
|
|
93677
93691
|
// src/output/text/index.ts
|
|
93678
|
-
var indexFormatters = [
|
|
93692
|
+
var indexFormatters = [
|
|
93693
|
+
{ command: "index", handler: (r) => formatIndexPlain(r) },
|
|
93694
|
+
{ command: "index-status", handler: (r) => formatIndexStatusPlain(r) }
|
|
93695
|
+
];
|
|
93679
93696
|
|
|
93680
93697
|
// src/output/text/info.ts
|
|
93681
93698
|
var infoFormatters = [{ command: "info", handler: (r) => formatInfoPlain(r) }];
|
|
@@ -94984,6 +95001,9 @@ function runMigrations2(db, options) {
|
|
|
94984
95001
|
const initialMigration = STATE_MIGRATIONS[0];
|
|
94985
95002
|
if (!initialMigration)
|
|
94986
95003
|
throw new Error("State migration registry has no initial migration.");
|
|
95004
|
+
const finalMigration = STATE_MIGRATIONS[STATE_MIGRATIONS.length - 1];
|
|
95005
|
+
if (!finalMigration)
|
|
95006
|
+
throw new Error("State migration registry has no final migration.");
|
|
94987
95007
|
let existingUnversionedSnapshotPrepared = false;
|
|
94988
95008
|
const prepareExistingUnversionedState = (lockedDb) => {
|
|
94989
95009
|
if (existingUnversionedSnapshotPrepared)
|
|
@@ -95005,7 +95025,7 @@ function runMigrations2(db, options) {
|
|
|
95005
95025
|
existingUnversionedSnapshotPrepared = true;
|
|
95006
95026
|
};
|
|
95007
95027
|
runMigrations(db, STATE_MIGRATIONS, {
|
|
95008
|
-
lockInitialMigrationPrefixThrough: options?.existingUnversionedDatabase ? "002-task-history-per-run" : undefined,
|
|
95028
|
+
lockInitialMigrationPrefixThrough: options?.existingUnversionedDatabase ? "002-task-history-per-run" : options?.freshDatabase ? finalMigration.id : undefined,
|
|
95009
95029
|
beforeLedgerInitializationLocked(lockedDb) {
|
|
95010
95030
|
if (options?.freshDatabase)
|
|
95011
95031
|
return;
|
|
@@ -95266,10 +95286,13 @@ function openStateDatabase(dbPath, options) {
|
|
|
95266
95286
|
});
|
|
95267
95287
|
try {
|
|
95268
95288
|
preflight.exec("PRAGMA busy_timeout = 30000");
|
|
95269
|
-
const ledger =
|
|
95289
|
+
const { ledger, hasNoOtherTables } = preflight.transaction(() => ({
|
|
95290
|
+
ledger: assertMigrationLedger(preflight, STATE_MIGRATIONS),
|
|
95291
|
+
hasNoOtherTables: unversionedDatabaseHasNoTables(preflight)
|
|
95292
|
+
}))();
|
|
95270
95293
|
warnNewerStateLedger(ledger);
|
|
95271
95294
|
existingUnversionedDatabase = ledger.migrationIds.length === 0;
|
|
95272
|
-
if (existingUnversionedDatabase &&
|
|
95295
|
+
if (existingUnversionedDatabase && hasNoOtherTables) {
|
|
95273
95296
|
existingUnversionedDatabase = false;
|
|
95274
95297
|
treatUnversionedAsFresh = true;
|
|
95275
95298
|
}
|
|
@@ -95351,7 +95374,7 @@ function openStateDatabase(dbPath, options) {
|
|
|
95351
95374
|
if (freshReservation)
|
|
95352
95375
|
closeFileIdentity(freshReservation);
|
|
95353
95376
|
releaseActivity?.();
|
|
95354
|
-
|
|
95377
|
+
throwBeginFailure(error, "state");
|
|
95355
95378
|
}
|
|
95356
95379
|
}
|
|
95357
95380
|
function listPendingStateMigrations(dbPath = getStateDbPath()) {
|
|
@@ -95405,15 +95428,15 @@ function sleepSyncMs2(ms) {
|
|
|
95405
95428
|
return;
|
|
95406
95429
|
sleepSync(ms);
|
|
95407
95430
|
}
|
|
95408
|
-
function throwBeginFailure(err) {
|
|
95431
|
+
function throwBeginFailure(err, dbKind) {
|
|
95409
95432
|
if (isSqliteContentionError(err)) {
|
|
95410
|
-
const contended = new TransientError("akm's state database is busy (another akm process is writing it); retry shortly.", "STATE_DB_CONTENDED");
|
|
95433
|
+
const contended = dbKind === "index" ? new TransientError("akm's index database is busy (another akm process is writing it); retry shortly.", "INDEX_DB_CONTENDED") : new TransientError("akm's state database is busy (another akm process is writing it); retry shortly.", "STATE_DB_CONTENDED");
|
|
95411
95434
|
contended.cause = err;
|
|
95412
95435
|
throw contended;
|
|
95413
95436
|
}
|
|
95414
95437
|
throw err;
|
|
95415
95438
|
}
|
|
95416
|
-
function beginImmediateTransaction(db) {
|
|
95439
|
+
function beginImmediateTransaction(db, dbKind = "state") {
|
|
95417
95440
|
if (db.inTransaction) {
|
|
95418
95441
|
throw new Error("beginImmediateTransaction requires a connection with no active transaction");
|
|
95419
95442
|
}
|
|
@@ -95436,16 +95459,16 @@ function beginImmediateTransaction(db) {
|
|
|
95436
95459
|
sleepSyncMs2(2 ** (attempt - 1));
|
|
95437
95460
|
continue;
|
|
95438
95461
|
}
|
|
95439
|
-
throwBeginFailure(err);
|
|
95462
|
+
throwBeginFailure(err, dbKind);
|
|
95440
95463
|
}
|
|
95441
95464
|
}
|
|
95442
|
-
throwBeginFailure(lastBeginErr);
|
|
95465
|
+
throwBeginFailure(lastBeginErr, dbKind);
|
|
95443
95466
|
}
|
|
95444
|
-
function withImmediateTransaction(db, fn) {
|
|
95467
|
+
function withImmediateTransaction(db, fn, dbKind = "state") {
|
|
95445
95468
|
if (db.inTransaction) {
|
|
95446
95469
|
return fn();
|
|
95447
95470
|
}
|
|
95448
|
-
beginImmediateTransaction(db);
|
|
95471
|
+
beginImmediateTransaction(db, dbKind);
|
|
95449
95472
|
try {
|
|
95450
95473
|
const result = fn();
|
|
95451
95474
|
if (!db.inTransaction) {
|
|
@@ -95781,9 +95804,9 @@ function listTxnJournalsTolerant(predicate) {
|
|
|
95781
95804
|
|
|
95782
95805
|
// src/commands/proposal/repository.ts
|
|
95783
95806
|
import { createHash as createHash11, randomUUID as randomUUID6 } from "node:crypto";
|
|
95784
|
-
import
|
|
95807
|
+
import fs53 from "node:fs";
|
|
95785
95808
|
init_dist();
|
|
95786
|
-
import
|
|
95809
|
+
import path65 from "node:path";
|
|
95787
95810
|
|
|
95788
95811
|
// src/core/adapter/validate-context.ts
|
|
95789
95812
|
init_asset_placement();
|
|
@@ -95842,20 +95865,9 @@ init_warn();
|
|
|
95842
95865
|
init_write_provenance();
|
|
95843
95866
|
|
|
95844
95867
|
// src/indexer/index-written-assets.ts
|
|
95845
|
-
import
|
|
95846
|
-
import path64 from "node:path";
|
|
95868
|
+
import path63 from "node:path";
|
|
95847
95869
|
init_errors();
|
|
95848
95870
|
init_paths();
|
|
95849
|
-
|
|
95850
|
-
// src/core/run-lock.ts
|
|
95851
|
-
init_errors();
|
|
95852
|
-
function formatLockHolderPid(holder) {
|
|
95853
|
-
if (holder.pid === null)
|
|
95854
|
-
return "unknown";
|
|
95855
|
-
return holder.launcherPid !== null ? `${holder.pid} (launcher ${holder.launcherPid})` : String(holder.pid);
|
|
95856
|
-
}
|
|
95857
|
-
|
|
95858
|
-
// src/indexer/index-written-assets.ts
|
|
95859
95871
|
init_warn();
|
|
95860
95872
|
|
|
95861
95873
|
// src/storage/repositories/index-connection.ts
|
|
@@ -95864,7 +95876,7 @@ init_paths();
|
|
|
95864
95876
|
init_warn();
|
|
95865
95877
|
|
|
95866
95878
|
// src/storage/repositories/index-entry-schema.ts
|
|
95867
|
-
var CANONICAL_INDEX_DB_VERSION =
|
|
95879
|
+
var CANONICAL_INDEX_DB_VERSION = 24;
|
|
95868
95880
|
var CANONICAL_ENTRY_SCHEMA_FINGERPRINT = {
|
|
95869
95881
|
tableSql: "CREATE TABLE entries ( id INTEGER PRIMARY KEY AUTOINCREMENT, item_ref TEXT NOT NULL UNIQUE, bundle_id TEXT NOT NULL, component_id TEXT NOT NULL, concept_id TEXT NOT NULL, adapter_id TEXT NOT NULL, type TEXT NOT NULL, file_path TEXT NOT NULL, content_hash TEXT, document_json TEXT NOT NULL, search_text TEXT NOT NULL, derived_from TEXT )",
|
|
95870
95882
|
sqliteSequenceTable: true,
|
|
@@ -96016,9 +96028,7 @@ var CANONICAL_ENTRY_SCHEMA_FINGERPRINT = {
|
|
|
96016
96028
|
}
|
|
96017
96029
|
],
|
|
96018
96030
|
searchSurfaces: {
|
|
96019
|
-
|
|
96020
|
-
fragmentSourceSql: "CREATE TABLE entry_fragments ( entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE, safe_markdown TEXT NOT NULL )",
|
|
96021
|
-
fragmentsFtsSql: "CREATE VIRTUAL TABLE entry_fragments_fts USING fts5( entry_id UNINDEXED, fragment_id UNINDEXED, fragment_ordinal UNINDEXED, content, tokenize='porter unicode61' )"
|
|
96031
|
+
fragmentSourceSql: "CREATE TABLE entry_fragments ( entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE, safe_markdown TEXT NOT NULL )"
|
|
96022
96032
|
}
|
|
96023
96033
|
};
|
|
96024
96034
|
function sqlString(value) {
|
|
@@ -96068,9 +96078,7 @@ function readEntrySchemaFingerprint(db) {
|
|
|
96068
96078
|
columns,
|
|
96069
96079
|
indexes,
|
|
96070
96080
|
searchSurfaces: {
|
|
96071
|
-
|
|
96072
|
-
fragmentSourceSql: readNamedTableSql(db, "entry_fragments"),
|
|
96073
|
-
fragmentsFtsSql: readNamedTableSql(db, "entry_fragments_fts")
|
|
96081
|
+
fragmentSourceSql: readNamedTableSql(db, "entry_fragments")
|
|
96074
96082
|
}
|
|
96075
96083
|
};
|
|
96076
96084
|
}
|
|
@@ -96110,13 +96118,66 @@ function isCanonicalIndexGeneration(db) {
|
|
|
96110
96118
|
init_errors();
|
|
96111
96119
|
init_warn();
|
|
96112
96120
|
|
|
96113
|
-
// src/storage/repositories/
|
|
96114
|
-
|
|
96121
|
+
// src/storage/repositories/index-sql.ts
|
|
96122
|
+
var SQLITE_CHUNK_SIZE = 500;
|
|
96115
96123
|
|
|
96116
|
-
// src/storage/repositories/
|
|
96117
|
-
function
|
|
96118
|
-
|
|
96119
|
-
|
|
96124
|
+
// src/storage/repositories/files-repository.ts
|
|
96125
|
+
function rowToFileState(row) {
|
|
96126
|
+
return {
|
|
96127
|
+
path: row.path,
|
|
96128
|
+
bundleId: row.bundle_id,
|
|
96129
|
+
size: row.size,
|
|
96130
|
+
mtimeMs: row.mtime_ms,
|
|
96131
|
+
ctimeMs: row.ctime_ms,
|
|
96132
|
+
blobHash: row.blob_hash,
|
|
96133
|
+
adapterId: row.adapter_id
|
|
96134
|
+
};
|
|
96135
|
+
}
|
|
96136
|
+
function getFileState(db, path58) {
|
|
96137
|
+
const row = db.prepare("SELECT path, bundle_id, size, mtime_ms, ctime_ms, blob_hash, adapter_id FROM files WHERE path = ?").get(path58);
|
|
96138
|
+
return row ? rowToFileState(row) : undefined;
|
|
96139
|
+
}
|
|
96140
|
+
function upsertFileState(db, row) {
|
|
96141
|
+
db.prepare(`INSERT INTO files (path, bundle_id, size, mtime_ms, ctime_ms, blob_hash, adapter_id) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
96142
|
+
ON CONFLICT(path) DO UPDATE SET bundle_id = excluded.bundle_id, size = excluded.size,
|
|
96143
|
+
mtime_ms = excluded.mtime_ms, ctime_ms = excluded.ctime_ms, blob_hash = excluded.blob_hash,
|
|
96144
|
+
adapter_id = excluded.adapter_id`).run(row.path, row.bundleId, row.size, row.mtimeMs, row.ctimeMs, row.blobHash, row.adapterId);
|
|
96145
|
+
}
|
|
96146
|
+
function deleteFileStates(db, paths) {
|
|
96147
|
+
for (let offset = 0;offset < paths.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96148
|
+
const chunk = paths.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96149
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
96150
|
+
db.prepare(`DELETE FROM files WHERE path IN (${placeholders})`).run(...chunk);
|
|
96151
|
+
}
|
|
96152
|
+
}
|
|
96153
|
+
function insertNewUnitTexts(db, units) {
|
|
96154
|
+
if (units.length === 0)
|
|
96155
|
+
return { inserted: 0 };
|
|
96156
|
+
const insertText = db.prepare("INSERT OR IGNORE INTO unit_texts (unit_hash, kind, text) VALUES (?, ?, ?)");
|
|
96157
|
+
const insertFts = db.prepare("INSERT INTO units_fts (unit_hash, text) VALUES (?, ?)");
|
|
96158
|
+
let inserted = 0;
|
|
96159
|
+
for (const unit of units) {
|
|
96160
|
+
const result = insertText.run(unit.hash, unit.kind, unit.text);
|
|
96161
|
+
if (Number(result.changes) > 0) {
|
|
96162
|
+
insertFts.run(unit.hash, unit.text);
|
|
96163
|
+
inserted++;
|
|
96164
|
+
}
|
|
96165
|
+
}
|
|
96166
|
+
return { inserted };
|
|
96167
|
+
}
|
|
96168
|
+
function pruneOrphanUnitTextsForHashes(db, hashes) {
|
|
96169
|
+
const unique = [...new Set(hashes)];
|
|
96170
|
+
if (unique.length === 0)
|
|
96171
|
+
return { removed: 0 };
|
|
96172
|
+
let removed = 0;
|
|
96173
|
+
for (let offset = 0;offset < unique.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96174
|
+
const chunk = unique.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96175
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
96176
|
+
db.prepare(`DELETE FROM units_fts WHERE unit_hash IN (${placeholders}) AND NOT EXISTS ` + `(SELECT 1 FROM entry_units WHERE entry_units.unit_hash = units_fts.unit_hash)`).run(...chunk);
|
|
96177
|
+
const result = db.prepare(`DELETE FROM unit_texts WHERE unit_hash IN (${placeholders}) AND NOT EXISTS ` + `(SELECT 1 FROM entry_units WHERE entry_units.unit_hash = unit_texts.unit_hash)`).run(...chunk);
|
|
96178
|
+
removed += Number(result.changes);
|
|
96179
|
+
}
|
|
96180
|
+
return { removed };
|
|
96120
96181
|
}
|
|
96121
96182
|
|
|
96122
96183
|
// src/storage/repositories/index-meta-repository.ts
|
|
@@ -96127,112 +96188,16 @@ function getMeta(db, key) {
|
|
|
96127
96188
|
function setMeta(db, key, value) {
|
|
96128
96189
|
db.prepare("INSERT OR REPLACE INTO index_meta (key, value) VALUES (?, ?)").run(key, value);
|
|
96129
96190
|
}
|
|
96130
|
-
function deleteMeta(db, key) {
|
|
96131
|
-
db.prepare("DELETE FROM index_meta WHERE key = ?").run(key);
|
|
96132
|
-
}
|
|
96133
|
-
|
|
96134
|
-
// src/storage/repositories/index-sql.ts
|
|
96135
|
-
var SQLITE_CHUNK_SIZE = 500;
|
|
96136
|
-
|
|
96137
|
-
// src/storage/repositories/embedding-salvage-repository.ts
|
|
96138
|
-
function hashEmbeddableText(searchText) {
|
|
96139
|
-
return createHash9("sha256").update(searchText, "utf8").digest("hex");
|
|
96140
|
-
}
|
|
96141
|
-
function purgeEmbeddingSalvage(db) {
|
|
96142
|
-
db.exec("DELETE FROM embedding_salvage");
|
|
96143
|
-
}
|
|
96144
|
-
function relabelEmbeddingSalvageFingerprint(db, fromFingerprint, toFingerprint) {
|
|
96145
|
-
db.prepare("UPDATE embedding_salvage SET fingerprint = ? WHERE fingerprint = ?").run(toFingerprint, fromFingerprint);
|
|
96146
|
-
}
|
|
96147
|
-
function reuseSalvagedEmbeddings(db, entries, fingerprint, writeReused) {
|
|
96148
|
-
if (entries.length === 0)
|
|
96149
|
-
return { reusedCount: 0, remaining: [] };
|
|
96150
|
-
const anySalvageForFingerprint = db.prepare("SELECT 1 FROM embedding_salvage WHERE fingerprint = ? LIMIT 1").get(fingerprint);
|
|
96151
|
-
if (!anySalvageForFingerprint)
|
|
96152
|
-
return { reusedCount: 0, remaining: [...entries] };
|
|
96153
|
-
const hashes = entries.map((entry) => hashEmbeddableText(entry.searchText));
|
|
96154
|
-
const salvageByHash = new Map;
|
|
96155
|
-
const uniqueHashes = [...new Set(hashes)];
|
|
96156
|
-
for (let offset = 0;offset < uniqueHashes.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96157
|
-
const chunk = uniqueHashes.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96158
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
96159
|
-
const rows = db.prepare(`SELECT content_hash AS contentHash, embedding FROM embedding_salvage WHERE fingerprint = ? AND content_hash IN (${placeholders})`).all(fingerprint, ...chunk);
|
|
96160
|
-
for (const row of rows)
|
|
96161
|
-
salvageByHash.set(row.contentHash, row.embedding);
|
|
96162
|
-
}
|
|
96163
|
-
if (salvageByHash.size === 0)
|
|
96164
|
-
return { reusedCount: 0, remaining: [...entries] };
|
|
96165
|
-
let reusedCount = 0;
|
|
96166
|
-
const remaining = [];
|
|
96167
|
-
for (let offset = 0;offset < entries.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96168
|
-
const end = Math.min(offset + SQLITE_CHUNK_SIZE, entries.length);
|
|
96169
|
-
const chunkMatches = [];
|
|
96170
|
-
for (let i = offset;i < end; i++) {
|
|
96171
|
-
const entry = entries[i];
|
|
96172
|
-
const blob = salvageByHash.get(hashes[i]);
|
|
96173
|
-
if (blob)
|
|
96174
|
-
chunkMatches.push({ entry, blob });
|
|
96175
|
-
else
|
|
96176
|
-
remaining.push(entry);
|
|
96177
|
-
}
|
|
96178
|
-
if (chunkMatches.length === 0)
|
|
96179
|
-
continue;
|
|
96180
|
-
db.transaction(() => {
|
|
96181
|
-
for (const { entry, blob } of chunkMatches) {
|
|
96182
|
-
if (writeReused(entry, blobToEmbedding(blob)))
|
|
96183
|
-
reusedCount++;
|
|
96184
|
-
else
|
|
96185
|
-
remaining.push(entry);
|
|
96186
|
-
}
|
|
96187
|
-
})();
|
|
96188
|
-
}
|
|
96189
|
-
return { reusedCount, remaining };
|
|
96190
|
-
}
|
|
96191
96191
|
|
|
96192
96192
|
// src/storage/repositories/index-vec-repository.ts
|
|
96193
96193
|
import { createRequire as createRequire4 } from "node:module";
|
|
96194
|
-
|
|
96195
|
-
// src/core/best-effort.ts
|
|
96196
|
-
init_warn();
|
|
96197
|
-
function bestEffort(fn, context) {
|
|
96198
|
-
try {
|
|
96199
|
-
return fn();
|
|
96200
|
-
} catch (err) {
|
|
96201
|
-
if (isVerbose()) {
|
|
96202
|
-
warnVerbose(`[akm:best-effort] ${context ? `${context}: ` : ""}swallowed error`, err);
|
|
96203
|
-
}
|
|
96204
|
-
return;
|
|
96205
|
-
}
|
|
96206
|
-
}
|
|
96207
|
-
|
|
96208
|
-
// src/storage/repositories/index-vec-repository.ts
|
|
96209
|
-
init_warn();
|
|
96210
|
-
|
|
96211
|
-
// src/llm/embedders/types.ts
|
|
96212
|
-
init_warn();
|
|
96213
|
-
function cosineSimilarity(a, b) {
|
|
96214
|
-
if (a.length !== b.length) {
|
|
96215
|
-
warn("cosineSimilarity: vector dimension mismatch (%d vs %d) — re-index recommended", a.length, b.length);
|
|
96216
|
-
return 0;
|
|
96217
|
-
}
|
|
96218
|
-
const len = a.length;
|
|
96219
|
-
if (len === 0)
|
|
96220
|
-
return 0;
|
|
96221
|
-
let dot = 0;
|
|
96222
|
-
let magA = 0;
|
|
96223
|
-
let magB = 0;
|
|
96224
|
-
for (let i = 0;i < len; i++) {
|
|
96225
|
-
dot += a[i] * b[i];
|
|
96226
|
-
magA += a[i] * a[i];
|
|
96227
|
-
magB += b[i] * b[i];
|
|
96228
|
-
}
|
|
96229
|
-
const denom = Math.sqrt(magA) * Math.sqrt(magB);
|
|
96230
|
-
return denom === 0 ? 0 : dot / denom;
|
|
96231
|
-
}
|
|
96232
|
-
|
|
96233
|
-
// src/storage/repositories/index-vec-repository.ts
|
|
96234
96194
|
var vecStatus = new WeakMap;
|
|
96195
|
+
var forceVecUnavailableForTests = false;
|
|
96235
96196
|
function loadVecExtension(db) {
|
|
96197
|
+
if (forceVecUnavailableForTests) {
|
|
96198
|
+
vecStatus.set(db, false);
|
|
96199
|
+
return;
|
|
96200
|
+
}
|
|
96236
96201
|
try {
|
|
96237
96202
|
const esmRequire = createRequire4(import.meta.url);
|
|
96238
96203
|
const sqliteVec = esmRequire("sqlite-vec");
|
|
@@ -96245,135 +96210,135 @@ function loadVecExtension(db) {
|
|
|
96245
96210
|
function isVecAvailable(db) {
|
|
96246
96211
|
return vecStatus.get(db) ?? false;
|
|
96247
96212
|
}
|
|
96248
|
-
|
|
96249
|
-
|
|
96250
|
-
|
|
96213
|
+
|
|
96214
|
+
// src/storage/repositories/units-repository.ts
|
|
96215
|
+
init_errors();
|
|
96216
|
+
function createUnitsVecTable(db, dim) {
|
|
96217
|
+
db.exec(`
|
|
96218
|
+
CREATE VIRTUAL TABLE units_vec USING vec0(
|
|
96219
|
+
unit_id INTEGER PRIMARY KEY,
|
|
96220
|
+
embedding FLOAT[${dim}],
|
|
96221
|
+
+unit_hash TEXT,
|
|
96222
|
+
+identity TEXT
|
|
96223
|
+
);
|
|
96224
|
+
`);
|
|
96251
96225
|
}
|
|
96252
|
-
function
|
|
96253
|
-
|
|
96254
|
-
return false;
|
|
96255
|
-
return hasVecTable(db);
|
|
96226
|
+
function tableExists(db, name) {
|
|
96227
|
+
return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) != null;
|
|
96256
96228
|
}
|
|
96257
|
-
function
|
|
96258
|
-
|
|
96259
|
-
|
|
96260
|
-
|
|
96261
|
-
const missingVecRows = db.prepare(`
|
|
96262
|
-
SELECT id FROM embeddings
|
|
96263
|
-
EXCEPT
|
|
96264
|
-
SELECT id FROM entries_vec
|
|
96265
|
-
LIMIT 1
|
|
96266
|
-
`).all();
|
|
96267
|
-
if (missingVecRows.length > 0)
|
|
96268
|
-
return false;
|
|
96269
|
-
const orphanVecRows = db.prepare(`
|
|
96270
|
-
SELECT id FROM entries_vec
|
|
96271
|
-
EXCEPT
|
|
96272
|
-
SELECT id FROM embeddings
|
|
96273
|
-
LIMIT 1
|
|
96274
|
-
`).all();
|
|
96275
|
-
return orphanVecRows.length === 0;
|
|
96276
|
-
} catch {
|
|
96277
|
-
return false;
|
|
96278
|
-
}
|
|
96229
|
+
function unitsVecDimension(db) {
|
|
96230
|
+
const row = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'units_vec'").get();
|
|
96231
|
+
const match = row?.sql.match(/FLOAT\[(\d+)\]/);
|
|
96232
|
+
return match?.[1] === undefined ? undefined : Number(match[1]);
|
|
96279
96233
|
}
|
|
96280
|
-
|
|
96281
|
-
|
|
96282
|
-
|
|
96283
|
-
|
|
96284
|
-
|
|
96285
|
-
|
|
96286
|
-
|
|
96287
|
-
|
|
96288
|
-
|
|
96289
|
-
|
|
96290
|
-
|
|
96291
|
-
|
|
96292
|
-
|
|
96293
|
-
|
|
96294
|
-
|
|
96295
|
-
|
|
96296
|
-
|
|
96297
|
-
db.prepare("DELETE FROM entries_vec WHERE id = ?").run(id);
|
|
96298
|
-
}
|
|
96299
|
-
var vecInitWarnedDbs = new WeakSet;
|
|
96300
|
-
function purgeEmbeddings(db, opts) {
|
|
96301
|
-
bestEffort(() => db.exec("DELETE FROM embeddings"), "purge embeddings");
|
|
96302
|
-
if (isVecAvailable(db)) {
|
|
96303
|
-
bestEffort(() => db.exec(opts?.dropVecTable ? "DROP TABLE IF EXISTS entries_vec" : "DELETE FROM entries_vec"), "purge entries_vec");
|
|
96304
|
-
}
|
|
96305
|
-
setMeta(db, "hasEmbeddings", "0");
|
|
96306
|
-
}
|
|
96307
|
-
function upsertEmbedding(db, entryId, embedding) {
|
|
96308
|
-
const exists = db.prepare("SELECT 1 FROM entries WHERE id = ?").get(entryId);
|
|
96309
|
-
if (!exists)
|
|
96310
|
-
return { stored: false, vec: "unavailable" };
|
|
96311
|
-
const buf = float32Buffer(embedding);
|
|
96312
|
-
db.prepare("INSERT OR REPLACE INTO embeddings (id, embedding) VALUES (?, ?)").run(entryId, buf);
|
|
96234
|
+
function ensureUnitTables(db, dim) {
|
|
96235
|
+
db.exec(`
|
|
96236
|
+
CREATE TABLE IF NOT EXISTS units (
|
|
96237
|
+
unit_id INTEGER PRIMARY KEY,
|
|
96238
|
+
unit_hash TEXT NOT NULL,
|
|
96239
|
+
identity TEXT NOT NULL,
|
|
96240
|
+
UNIQUE (unit_hash, identity)
|
|
96241
|
+
);
|
|
96242
|
+
CREATE TABLE IF NOT EXISTS entry_units (
|
|
96243
|
+
entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE,
|
|
96244
|
+
ordinal INTEGER NOT NULL,
|
|
96245
|
+
fragment_id TEXT,
|
|
96246
|
+
unit_hash TEXT NOT NULL,
|
|
96247
|
+
PRIMARY KEY (entry_id, ordinal)
|
|
96248
|
+
);
|
|
96249
|
+
CREATE INDEX IF NOT EXISTS entry_units_hash ON entry_units(unit_hash);
|
|
96250
|
+
`);
|
|
96313
96251
|
if (!isVecAvailable(db))
|
|
96314
|
-
return
|
|
96315
|
-
|
|
96316
|
-
|
|
96317
|
-
|
|
96318
|
-
|
|
96319
|
-
|
|
96320
|
-
|
|
96321
|
-
|
|
96322
|
-
|
|
96252
|
+
return;
|
|
96253
|
+
if (tableExists(db, "units_vec"))
|
|
96254
|
+
return;
|
|
96255
|
+
createUnitsVecTable(db, dim);
|
|
96256
|
+
}
|
|
96257
|
+
function float32Buffer(vector) {
|
|
96258
|
+
return Buffer.from(new Float32Array(vector).buffer);
|
|
96259
|
+
}
|
|
96260
|
+
function upsertUnitVectors(db, rows) {
|
|
96261
|
+
if (rows.length === 0 || !isVecAvailable(db))
|
|
96262
|
+
return { inserted: 0, failed: 0 };
|
|
96263
|
+
const insertUnit = db.prepare("INSERT INTO units (unit_hash, identity) VALUES (?, ?) ON CONFLICT(unit_hash, identity) DO NOTHING");
|
|
96264
|
+
const selectUnitId = db.prepare("SELECT unit_id FROM units WHERE unit_hash = ? AND identity = ?");
|
|
96265
|
+
const deleteVec = db.prepare("DELETE FROM units_vec WHERE unit_id = ?");
|
|
96266
|
+
const insertVec = db.prepare("INSERT INTO units_vec (unit_id, embedding, unit_hash, identity) VALUES (?, ?, ?, ?)");
|
|
96267
|
+
const deleteUnit = db.prepare("DELETE FROM units WHERE unit_id = ?");
|
|
96268
|
+
const writeOne = (row) => withImmediateTransaction(db, () => {
|
|
96269
|
+
const result = insertUnit.run(row.hash, row.identity);
|
|
96270
|
+
const freshlyInserted = Number(result.changes) > 0;
|
|
96271
|
+
const unitRow = selectUnitId.get(row.hash, row.identity);
|
|
96272
|
+
if (!unitRow)
|
|
96273
|
+
return false;
|
|
96274
|
+
deleteVec.run(unitRow.unit_id);
|
|
96275
|
+
try {
|
|
96276
|
+
insertVec.run(unitRow.unit_id, float32Buffer(row.vector), row.hash, row.identity);
|
|
96277
|
+
} catch (err) {
|
|
96278
|
+
deleteUnit.run(unitRow.unit_id);
|
|
96279
|
+
throw err;
|
|
96280
|
+
}
|
|
96281
|
+
return freshlyInserted;
|
|
96282
|
+
}, "index");
|
|
96283
|
+
let inserted = 0;
|
|
96284
|
+
let failed = 0;
|
|
96285
|
+
for (const row of rows) {
|
|
96286
|
+
try {
|
|
96287
|
+
if (writeOne(row))
|
|
96288
|
+
inserted++;
|
|
96289
|
+
} catch {
|
|
96290
|
+
failed++;
|
|
96291
|
+
}
|
|
96323
96292
|
}
|
|
96293
|
+
return { inserted, failed };
|
|
96324
96294
|
}
|
|
96325
|
-
function
|
|
96326
|
-
const
|
|
96327
|
-
|
|
96328
|
-
|
|
96329
|
-
|
|
96330
|
-
|
|
96331
|
-
|
|
96332
|
-
return null;
|
|
96333
|
-
}
|
|
96334
|
-
const aligned = new ArrayBuffer(buf.byteLength);
|
|
96335
|
-
new Uint8Array(aligned).set(buf);
|
|
96336
|
-
const f32 = new Float32Array(aligned);
|
|
96337
|
-
return Array.from(f32);
|
|
96338
|
-
}
|
|
96339
|
-
function getAllEntriesForEmbedding(db, entryIds) {
|
|
96340
|
-
const select = `
|
|
96341
|
-
SELECT e.id, e.search_text AS searchText, e.item_ref AS itemRef, e.file_path AS filePath FROM entries e
|
|
96342
|
-
`;
|
|
96343
|
-
const missing = "NOT EXISTS (SELECT 1 FROM embeddings b WHERE b.id = e.id)";
|
|
96344
|
-
if (entryIds === undefined) {
|
|
96345
|
-
return db.prepare(`${select} WHERE ${missing} ORDER BY e.id`).all();
|
|
96346
|
-
}
|
|
96347
|
-
const targets = [...new Set(entryIds)].sort((left, right) => left - right);
|
|
96348
|
-
const rows = [];
|
|
96349
|
-
for (let offset = 0;offset < targets.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96350
|
-
const chunk = targets.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96351
|
-
if (chunk.length === 0)
|
|
96352
|
-
continue;
|
|
96295
|
+
function listMissingHashes(db, hashes, identity3) {
|
|
96296
|
+
const unique = [...new Set(hashes)];
|
|
96297
|
+
if (unique.length === 0)
|
|
96298
|
+
return [];
|
|
96299
|
+
const present = new Set;
|
|
96300
|
+
for (let offset = 0;offset < unique.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96301
|
+
const chunk = unique.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96353
96302
|
const placeholders = chunk.map(() => "?").join(",");
|
|
96354
|
-
rows
|
|
96303
|
+
const rows = db.prepare(`SELECT unit_hash FROM units WHERE identity = ? AND unit_hash IN (${placeholders})`).all(identity3, ...chunk);
|
|
96304
|
+
for (const row of rows)
|
|
96305
|
+
present.add(row.unit_hash);
|
|
96355
96306
|
}
|
|
96356
|
-
return
|
|
96307
|
+
return unique.filter((hash3) => !present.has(hash3));
|
|
96357
96308
|
}
|
|
96358
|
-
function
|
|
96359
|
-
|
|
96360
|
-
|
|
96309
|
+
function dropOtherIdentities(db, keep, dim) {
|
|
96310
|
+
if (!isVecAvailable(db))
|
|
96311
|
+
return { removed: 0 };
|
|
96312
|
+
ensureUnitTables(db, dim);
|
|
96313
|
+
const currentDim = unitsVecDimension(db);
|
|
96314
|
+
const widthChanged = currentDim !== undefined && currentDim !== dim;
|
|
96315
|
+
const staleCount = db.prepare("SELECT COUNT(*) AS n FROM units WHERE identity != ?").get(keep).n;
|
|
96316
|
+
if (staleCount === 0 && !widthChanged)
|
|
96317
|
+
return { removed: 0 };
|
|
96318
|
+
withImmediateTransaction(db, () => {
|
|
96319
|
+
if (widthChanged) {
|
|
96320
|
+
db.exec("DROP TABLE IF EXISTS units_vec");
|
|
96321
|
+
createUnitsVecTable(db, dim);
|
|
96322
|
+
} else {
|
|
96323
|
+
const staleIds = db.prepare("SELECT unit_id FROM units WHERE identity != ?").all(keep).map((row) => row.unit_id);
|
|
96324
|
+
for (let offset = 0;offset < staleIds.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96325
|
+
const chunk = staleIds.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96326
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
96327
|
+
db.prepare(`DELETE FROM units_vec WHERE unit_id IN (${placeholders})`).run(...chunk);
|
|
96328
|
+
}
|
|
96329
|
+
}
|
|
96330
|
+
db.prepare("DELETE FROM units WHERE identity != ?").run(keep);
|
|
96331
|
+
}, "index");
|
|
96332
|
+
return { removed: staleCount };
|
|
96361
96333
|
}
|
|
96362
|
-
function
|
|
96363
|
-
|
|
96364
|
-
|
|
96365
|
-
|
|
96366
|
-
|
|
96367
|
-
|
|
96368
|
-
|
|
96369
|
-
|
|
96370
|
-
const samples = [];
|
|
96371
|
-
for (const row of rows) {
|
|
96372
|
-
const vector = bufferToFloat32(row.embedding, Math.floor(row.embedding.byteLength / 4));
|
|
96373
|
-
if (vector)
|
|
96374
|
-
samples.push({ id: row.id, searchText: row.searchText, vector });
|
|
96375
|
-
}
|
|
96376
|
-
return samples;
|
|
96334
|
+
function replaceEntryUnits(db, entryId, units) {
|
|
96335
|
+
withImmediateTransaction(db, () => {
|
|
96336
|
+
db.prepare("DELETE FROM entry_units WHERE entry_id = ?").run(entryId);
|
|
96337
|
+
const insert = db.prepare("INSERT INTO entry_units (entry_id, ordinal, fragment_id, unit_hash) VALUES (?, ?, ?, ?)");
|
|
96338
|
+
for (const unit of units) {
|
|
96339
|
+
insert.run(entryId, unit.ordinal, unit.fragmentId, unit.hash);
|
|
96340
|
+
}
|
|
96341
|
+
}, "index");
|
|
96377
96342
|
}
|
|
96378
96343
|
|
|
96379
96344
|
// src/storage/repositories/index-connection.ts
|
|
@@ -96422,6 +96387,21 @@ function closeDatabase(db) {
|
|
|
96422
96387
|
import fs46 from "node:fs";
|
|
96423
96388
|
init_asset_ref();
|
|
96424
96389
|
init_resolve_ref();
|
|
96390
|
+
|
|
96391
|
+
// src/core/best-effort.ts
|
|
96392
|
+
init_warn();
|
|
96393
|
+
function bestEffort(fn, context) {
|
|
96394
|
+
try {
|
|
96395
|
+
return fn();
|
|
96396
|
+
} catch (err) {
|
|
96397
|
+
if (isVerbose()) {
|
|
96398
|
+
warnVerbose(`[akm:best-effort] ${context ? `${context}: ` : ""}swallowed error`, err);
|
|
96399
|
+
}
|
|
96400
|
+
return;
|
|
96401
|
+
}
|
|
96402
|
+
}
|
|
96403
|
+
|
|
96404
|
+
// src/storage/repositories/index-entries-repository.ts
|
|
96425
96405
|
init_warn();
|
|
96426
96406
|
init_metadata();
|
|
96427
96407
|
|
|
@@ -96489,11 +96469,11 @@ init_warn();
|
|
|
96489
96469
|
|
|
96490
96470
|
// src/core/asset/markdown-fragments.ts
|
|
96491
96471
|
init_markdown();
|
|
96492
|
-
import { createHash as
|
|
96472
|
+
import { createHash as createHash9 } from "node:crypto";
|
|
96493
96473
|
var MARKDOWN_FRAGMENT_MAX_CHARS = 1600;
|
|
96494
96474
|
var MARKDOWN_FRAGMENT_PREFIX = "akm-fragment-";
|
|
96495
96475
|
function hash3(text) {
|
|
96496
|
-
return
|
|
96476
|
+
return createHash9("sha256").update(text).digest("hex");
|
|
96497
96477
|
}
|
|
96498
96478
|
function uniqueSlugs(body) {
|
|
96499
96479
|
const out = new Map;
|
|
@@ -96607,48 +96587,30 @@ function splitMarkdownFragments(body, maxChars = MARKDOWN_FRAGMENT_MAX_CHARS) {
|
|
|
96607
96587
|
}
|
|
96608
96588
|
|
|
96609
96589
|
// src/storage/repositories/index-fts-repository.ts
|
|
96610
|
-
|
|
96611
|
-
|
|
96612
|
-
|
|
96613
|
-
var ftsMutationStatementsByDb = new WeakMap;
|
|
96614
|
-
function getFtsMutationStatements(db) {
|
|
96615
|
-
const existing = ftsMutationStatementsByDb.get(db);
|
|
96590
|
+
var fragmentSourceStatementsByDb = new WeakMap;
|
|
96591
|
+
function getFragmentSourceStatements(db) {
|
|
96592
|
+
const existing = fragmentSourceStatementsByDb.get(db);
|
|
96616
96593
|
if (existing)
|
|
96617
96594
|
return existing;
|
|
96618
96595
|
const statements = {
|
|
96619
|
-
deleteOne: db.prepare("DELETE FROM entries_fts WHERE entry_id = ?"),
|
|
96620
|
-
insert: db.prepare(INSERT_FTS_SQL),
|
|
96621
|
-
deleteFragments: db.prepare("DELETE FROM entry_fragments_fts WHERE entry_id = ?"),
|
|
96622
96596
|
upsertFragmentSource: db.prepare("INSERT INTO entry_fragments (entry_id, safe_markdown) VALUES (?, ?) ON CONFLICT(entry_id) DO UPDATE SET safe_markdown = excluded.safe_markdown"),
|
|
96623
|
-
deleteFragmentSource: db.prepare("DELETE FROM entry_fragments WHERE entry_id = ?")
|
|
96624
|
-
insertFragment: db.prepare(INSERT_FRAGMENT_SQL)
|
|
96597
|
+
deleteFragmentSource: db.prepare("DELETE FROM entry_fragments WHERE entry_id = ?")
|
|
96625
96598
|
};
|
|
96626
|
-
|
|
96599
|
+
fragmentSourceStatementsByDb.set(db, statements);
|
|
96627
96600
|
return statements;
|
|
96628
96601
|
}
|
|
96629
|
-
function
|
|
96630
|
-
|
|
96631
|
-
const statements = getFtsMutationStatements(db);
|
|
96632
|
-
statements.deleteOne.run(entryId);
|
|
96633
|
-
statements.insert.run(entryId, fields.name, fields.description, fields.tags, fields.hints, fields.content);
|
|
96634
|
-
if (fragmentContent === undefined) {
|
|
96602
|
+
function replaceFragmentSource(db, entryId, fragmentContent) {
|
|
96603
|
+
if (fragmentContent === undefined)
|
|
96635
96604
|
return;
|
|
96636
|
-
|
|
96637
|
-
statements.deleteFragments.run(entryId);
|
|
96605
|
+
const statements = getFragmentSourceStatements(db);
|
|
96638
96606
|
statements.deleteFragmentSource.run(entryId);
|
|
96639
|
-
if (
|
|
96640
|
-
|
|
96641
|
-
statements.upsertFragmentSource.run(entryId, fragmentContent);
|
|
96642
|
-
for (const fragment of splitMarkdownFragments(fragmentContent)) {
|
|
96643
|
-
statements.insertFragment.run(entryId, fragment.fragmentId, fragment.ordinal, fragment.text.toLowerCase());
|
|
96644
|
-
}
|
|
96607
|
+
if (fragmentContent)
|
|
96608
|
+
statements.upsertFragmentSource.run(entryId, fragmentContent);
|
|
96645
96609
|
}
|
|
96646
|
-
function
|
|
96610
|
+
function deleteFragmentSource(db, entryIds) {
|
|
96647
96611
|
for (let i = 0;i < entryIds.length; i += SQLITE_CHUNK_SIZE) {
|
|
96648
96612
|
const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
|
|
96649
96613
|
const placeholders = chunk.map(() => "?").join(",");
|
|
96650
|
-
db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
|
|
96651
|
-
db.prepare(`DELETE FROM entry_fragments_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
|
|
96652
96614
|
db.prepare(`DELETE FROM entry_fragments WHERE entry_id IN (${placeholders})`).run(...chunk);
|
|
96653
96615
|
}
|
|
96654
96616
|
}
|
|
@@ -96658,16 +96620,13 @@ function upsertEntry(db, filePath, entry, searchText, provenance, contentHash) {
|
|
|
96658
96620
|
const stmts = getUpsertStmts(db);
|
|
96659
96621
|
const derivedFrom = typeof entry.derivedFrom === "string" && entry.derivedFrom.trim() ? entry.derivedFrom.trim() : null;
|
|
96660
96622
|
const apply = () => {
|
|
96661
|
-
const previous = stmts.findByItemRef.get(provenance.itemRef);
|
|
96662
96623
|
const result = stmts.upsert.get(provenance.itemRef, provenance.bundleId, provenance.componentId, provenance.conceptId, provenance.adapterId, entry.type, filePath, contentHash ?? null, JSON.stringify(entry), searchText, derivedFrom);
|
|
96663
96624
|
if (!result)
|
|
96664
96625
|
throw new Error("upsertEntry: item_ref not found after upsert");
|
|
96665
|
-
|
|
96666
|
-
deleteEntryVectors(db, result.id);
|
|
96667
|
-
replaceFtsEntry(db, result.id, entry, hasMarkdownFragmentContent(entry) ? getMarkdownFragmentContent(entry) ?? null : undefined);
|
|
96626
|
+
replaceFragmentSource(db, result.id, hasMarkdownFragmentContent(entry) ? getMarkdownFragmentContent(entry) ?? null : undefined);
|
|
96668
96627
|
return result.id;
|
|
96669
96628
|
};
|
|
96670
|
-
return db.transaction(apply)();
|
|
96629
|
+
return db.inTransaction ? db.transaction(apply)() : withImmediateTransaction(db, apply, "index");
|
|
96671
96630
|
}
|
|
96672
96631
|
var upsertStmtsByDb = new WeakMap;
|
|
96673
96632
|
var UPSERT_SET_CLAUSE = `SET
|
|
@@ -96694,8 +96653,7 @@ function getUpsertStmts(db) {
|
|
|
96694
96653
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
96695
96654
|
ON CONFLICT(item_ref) DO UPDATE ${UPSERT_SET_CLAUSE}
|
|
96696
96655
|
RETURNING id
|
|
96697
|
-
`)
|
|
96698
|
-
findByItemRef: db.prepare("SELECT id, search_text FROM entries WHERE item_ref = ?")
|
|
96656
|
+
`)
|
|
96699
96657
|
};
|
|
96700
96658
|
upsertStmtsByDb.set(db, stmts);
|
|
96701
96659
|
return stmts;
|
|
@@ -96704,15 +96662,10 @@ function deleteRelatedRows(db, ids, options = {}) {
|
|
|
96704
96662
|
if (ids.length === 0)
|
|
96705
96663
|
return;
|
|
96706
96664
|
const numericIds = ids.map((r) => r.id);
|
|
96707
|
-
|
|
96708
|
-
deleteFtsEntries(db, numericIds);
|
|
96665
|
+
deleteFragmentSource(db, numericIds);
|
|
96709
96666
|
for (let i = 0;i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
|
|
96710
96667
|
const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
|
|
96711
96668
|
const placeholders = chunk.map(() => "?").join(",");
|
|
96712
|
-
bestEffort(() => db.prepare(`DELETE FROM embeddings WHERE id IN (${placeholders})`).run(...chunk), "delete embeddings for entries");
|
|
96713
|
-
if (vecAvail) {
|
|
96714
|
-
bestEffort(() => db.prepare(`DELETE FROM entries_vec WHERE id IN (${placeholders})`).run(...chunk), "delete entries_vec for entries");
|
|
96715
|
-
}
|
|
96716
96669
|
bestEffort(() => db.prepare(`DELETE FROM utility_scores WHERE entry_id IN (${placeholders})`).run(...chunk), "delete utility_scores for entries");
|
|
96717
96670
|
bestEffort(() => db.prepare(`DELETE FROM utility_scores_scoped WHERE entry_id IN (${placeholders})`).run(...chunk), "delete utility_scores_scoped for entries");
|
|
96718
96671
|
}
|
|
@@ -96755,7 +96708,7 @@ function deleteUsageEventsByEntryIds(entryIds) {
|
|
|
96755
96708
|
function deleteEntriesByIds(db, ids) {
|
|
96756
96709
|
if (ids.length === 0)
|
|
96757
96710
|
return;
|
|
96758
|
-
db
|
|
96711
|
+
withImmediateTransaction(db, () => {
|
|
96759
96712
|
const idObjs = ids.map((id) => ({ id }));
|
|
96760
96713
|
deleteRelatedRows(db, idObjs);
|
|
96761
96714
|
for (let i = 0;i < ids.length; i += SQLITE_CHUNK_SIZE) {
|
|
@@ -96763,17 +96716,14 @@ function deleteEntriesByIds(db, ids) {
|
|
|
96763
96716
|
const placeholders = chunk.map(() => "?").join(",");
|
|
96764
96717
|
db.prepare(`DELETE FROM entries WHERE id IN (${placeholders})`).run(...chunk);
|
|
96765
96718
|
}
|
|
96766
|
-
})
|
|
96719
|
+
}, "index");
|
|
96767
96720
|
}
|
|
96768
96721
|
function getEntryCount(db) {
|
|
96769
96722
|
const row = db.prepare("SELECT COUNT(*) AS cnt FROM entries").get();
|
|
96770
96723
|
return row.cnt;
|
|
96771
96724
|
}
|
|
96772
|
-
function getEmbeddableEntryCount(db) {
|
|
96773
|
-
return getEntryCount(db);
|
|
96774
|
-
}
|
|
96775
96725
|
|
|
96776
|
-
// src/indexer/
|
|
96726
|
+
// src/indexer/drain.ts
|
|
96777
96727
|
init_paths();
|
|
96778
96728
|
init_warn();
|
|
96779
96729
|
|
|
@@ -97014,19 +96964,6 @@ var DEFAULT_TOKEN_BUDGET = 6000;
|
|
|
97014
96964
|
function estimateTokenCount(text) {
|
|
97015
96965
|
return Math.round(text.length / 4);
|
|
97016
96966
|
}
|
|
97017
|
-
var DEFAULT_MAX_INPUT_TOKENS = 512;
|
|
97018
|
-
function capEmbeddingText(text, maxTokens) {
|
|
97019
|
-
if (estimateTokenCount(text) <= maxTokens)
|
|
97020
|
-
return { text, truncated: false };
|
|
97021
|
-
const charBudget = Math.max(0, maxTokens * 4);
|
|
97022
|
-
let cut = Math.min(charBudget, text.length);
|
|
97023
|
-
if (cut > 0 && cut < text.length) {
|
|
97024
|
-
const code = text.charCodeAt(cut);
|
|
97025
|
-
if (code >= 56320 && code <= 57343)
|
|
97026
|
-
cut -= 1;
|
|
97027
|
-
}
|
|
97028
|
-
return { text: text.slice(0, cut), truncated: true };
|
|
97029
|
-
}
|
|
97030
96967
|
var DEFAULT_EMBEDDING_TIMEOUT_MS = 120000;
|
|
97031
96968
|
function resolveEmbeddingTimeoutMs(config) {
|
|
97032
96969
|
return config.timeoutMs ?? DEFAULT_EMBEDDING_TIMEOUT_MS;
|
|
@@ -97065,7 +97002,7 @@ function resolveEmbeddingConcurrency(config) {
|
|
|
97065
97002
|
return config.concurrency;
|
|
97066
97003
|
return defaultConcurrencyForEndpoint(config.endpoint);
|
|
97067
97004
|
}
|
|
97068
|
-
function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
|
|
97005
|
+
function buildTokenBoundedBatches(texts, tokenBudget, maxCount, tokenCounts) {
|
|
97069
97006
|
const batches = [];
|
|
97070
97007
|
let current = [];
|
|
97071
97008
|
let currentTokens = 0;
|
|
@@ -97077,7 +97014,7 @@ function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
|
|
|
97077
97014
|
}
|
|
97078
97015
|
};
|
|
97079
97016
|
for (let i = 0;i < texts.length; i++) {
|
|
97080
|
-
const tokens = estimateTokenCount(texts[i]);
|
|
97017
|
+
const tokens = tokenCounts?.[i] ?? estimateTokenCount(texts[i]);
|
|
97081
97018
|
if (tokens > tokenBudget) {
|
|
97082
97019
|
flush();
|
|
97083
97020
|
batches.push({ indices: [i], oversized: true });
|
|
@@ -97093,7 +97030,7 @@ function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
|
|
|
97093
97030
|
return batches;
|
|
97094
97031
|
}
|
|
97095
97032
|
var ADAPTIVE_BUDGET_SHRINK_FACTOR = 0.75;
|
|
97096
|
-
var
|
|
97033
|
+
var ADAPTIVE_BUDGET_FLOOR_TOKENS = 1024;
|
|
97097
97034
|
|
|
97098
97035
|
class RemoteEmbedder {
|
|
97099
97036
|
config;
|
|
@@ -97120,6 +97057,9 @@ class RemoteEmbedder {
|
|
|
97120
97057
|
if (ollamaOpts) {
|
|
97121
97058
|
body.options = ollamaOpts;
|
|
97122
97059
|
}
|
|
97060
|
+
if (isOllamaNativeEmbedEndpoint(this.endpoint)) {
|
|
97061
|
+
body.truncate = false;
|
|
97062
|
+
}
|
|
97123
97063
|
const timeoutMs = resolveEmbeddingTimeoutMs(this.config);
|
|
97124
97064
|
const response = await fetchWithTimeout(normalizeEmbeddingEndpoint(this.endpoint), {
|
|
97125
97065
|
method: "POST",
|
|
@@ -97140,30 +97080,31 @@ class RemoteEmbedder {
|
|
|
97140
97080
|
}
|
|
97141
97081
|
return l2Normalize(json.data[0].embedding);
|
|
97142
97082
|
}
|
|
97143
|
-
async embedBatch(texts, signal, onSkip, onBatch) {
|
|
97083
|
+
async embedBatch(texts, signal, onSkip, onBatch, packing) {
|
|
97144
97084
|
if (texts.length === 0)
|
|
97145
97085
|
return [];
|
|
97146
97086
|
const results = new Array(texts.length).fill(undefined);
|
|
97147
97087
|
const headers = this.buildHeaders();
|
|
97148
|
-
const ollamaOpts = resolveOllamaOptions(this.config);
|
|
97149
|
-
|
|
97150
|
-
|
|
97151
|
-
const
|
|
97152
|
-
const
|
|
97088
|
+
const ollamaOpts = resolveOllamaOptions(this.config, packing?.ollamaNumCtx);
|
|
97089
|
+
const tokenCounts = texts.map((text) => Math.ceil(text.length / (packing?.charsPerToken ?? 4)));
|
|
97090
|
+
let effectiveTokenBudget = packing?.tokenBudget ?? DEFAULT_TOKEN_BUDGET;
|
|
97091
|
+
const maxCount = packing?.maxCount ?? DEFAULT_REMOTE_BATCH_SIZE;
|
|
97092
|
+
const windowIsKnown = packing?.windowIsKnown ?? false;
|
|
97093
|
+
const textBatches = buildTokenBoundedBatches(texts, effectiveTokenBudget, maxCount, tokenCounts);
|
|
97153
97094
|
const configuredTimeoutMs = resolveEmbeddingTimeoutMs(this.config);
|
|
97154
97095
|
let dispatchedBatchCount = 0;
|
|
97155
|
-
let budgetShrunk =
|
|
97096
|
+
let budgetShrunk = windowIsKnown;
|
|
97156
97097
|
const maybeShrinkBudget = (rejectedIndices, rejectedBatchIndex, rejectedRequestTokens) => {
|
|
97157
97098
|
if (budgetShrunk)
|
|
97158
97099
|
return;
|
|
97159
97100
|
budgetShrunk = true;
|
|
97160
|
-
|
|
97161
|
-
effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), floor2);
|
|
97101
|
+
effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), ADAPTIVE_BUDGET_FLOOR_TOKENS);
|
|
97162
97102
|
const notYetDispatched = textBatches.slice(dispatchedBatchCount);
|
|
97163
97103
|
const remainingIndices = notYetDispatched.flatMap((batch) => batch.indices);
|
|
97164
97104
|
if (remainingIndices.length > 0) {
|
|
97165
97105
|
const remainingTexts = remainingIndices.map((i) => texts[i]);
|
|
97166
|
-
const
|
|
97106
|
+
const remainingCounts = remainingIndices.map((i) => tokenCounts[i]);
|
|
97107
|
+
const replanned = buildTokenBoundedBatches(remainingTexts, effectiveTokenBudget, maxCount, remainingCounts).map((batch) => ({
|
|
97167
97108
|
indices: batch.indices.map((localIndex) => remainingIndices[localIndex]),
|
|
97168
97109
|
oversized: batch.oversized
|
|
97169
97110
|
}));
|
|
@@ -97213,7 +97154,7 @@ class RemoteEmbedder {
|
|
|
97213
97154
|
if (dispatchAbort.signal.aborted)
|
|
97214
97155
|
return;
|
|
97215
97156
|
const batch = indices.map((i) => texts[i]);
|
|
97216
|
-
const requestTokens =
|
|
97157
|
+
const requestTokens = indices.reduce((sum, i) => sum + tokenCounts[i], 0);
|
|
97217
97158
|
const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, effectiveTokenBudget);
|
|
97218
97159
|
const requestStart = Date.now();
|
|
97219
97160
|
let batchEmbeddings;
|
|
@@ -97302,7 +97243,7 @@ class RemoteEmbedder {
|
|
|
97302
97243
|
dispatchedBatchCount = batchIndex;
|
|
97303
97244
|
if (textBatch.oversized) {
|
|
97304
97245
|
const idx = textBatch.indices[0];
|
|
97305
|
-
const estTokens =
|
|
97246
|
+
const estTokens = tokenCounts[idx];
|
|
97306
97247
|
onSkip?.({
|
|
97307
97248
|
index: idx,
|
|
97308
97249
|
reason: "context-window-exceeded",
|
|
@@ -97348,6 +97289,9 @@ class RemoteEmbedder {
|
|
|
97348
97289
|
if (ollamaOpts) {
|
|
97349
97290
|
body.options = ollamaOpts;
|
|
97350
97291
|
}
|
|
97292
|
+
if (isOllamaNativeEmbedEndpoint(this.endpoint)) {
|
|
97293
|
+
body.truncate = false;
|
|
97294
|
+
}
|
|
97351
97295
|
const response = await fetchWithTimeout(normalizeEmbeddingEndpoint(this.endpoint), {
|
|
97352
97296
|
method: "POST",
|
|
97353
97297
|
headers,
|
|
@@ -97415,6 +97359,15 @@ function normalizeEmbeddingEndpoint(endpoint) {
|
|
|
97415
97359
|
parsed.pathname = normalizedPath ? `${normalizedPath}/embeddings` : "/embeddings";
|
|
97416
97360
|
return parsed.toString();
|
|
97417
97361
|
}
|
|
97362
|
+
function isOllamaNativeEmbedEndpoint(endpoint) {
|
|
97363
|
+
let parsed;
|
|
97364
|
+
try {
|
|
97365
|
+
parsed = new URL(normalizeEmbeddingEndpoint(endpoint));
|
|
97366
|
+
} catch {
|
|
97367
|
+
return false;
|
|
97368
|
+
}
|
|
97369
|
+
return parsed.pathname.replace(/\/+$/, "").endsWith("/embed");
|
|
97370
|
+
}
|
|
97418
97371
|
function embeddingEndpointPathHint(endpoint) {
|
|
97419
97372
|
const normalizedEndpoint = normalizeEmbeddingEndpoint(endpoint);
|
|
97420
97373
|
if (normalizedEndpoint !== endpoint) {
|
|
@@ -97422,12 +97375,12 @@ function embeddingEndpointPathHint(endpoint) {
|
|
|
97422
97375
|
}
|
|
97423
97376
|
return "";
|
|
97424
97377
|
}
|
|
97425
|
-
function resolveOllamaOptions(config) {
|
|
97378
|
+
function resolveOllamaOptions(config, ollamaNumCtx) {
|
|
97426
97379
|
if (config.ollamaOptions && Object.keys(config.ollamaOptions).length > 0) {
|
|
97427
97380
|
return config.ollamaOptions;
|
|
97428
97381
|
}
|
|
97429
|
-
if (
|
|
97430
|
-
return { num_ctx:
|
|
97382
|
+
if (ollamaNumCtx) {
|
|
97383
|
+
return { num_ctx: ollamaNumCtx };
|
|
97431
97384
|
}
|
|
97432
97385
|
return;
|
|
97433
97386
|
}
|
|
@@ -97443,6 +97396,8 @@ function describeEmbeddingCredential(apiKey) {
|
|
|
97443
97396
|
return `${apiKey} (env)`;
|
|
97444
97397
|
return "literal apiKey";
|
|
97445
97398
|
}
|
|
97399
|
+
// src/llm/embedders/types.ts
|
|
97400
|
+
init_warn();
|
|
97446
97401
|
|
|
97447
97402
|
// src/llm/embedder.ts
|
|
97448
97403
|
var embedderOverrides;
|
|
@@ -97453,9 +97408,9 @@ function getLocalEmbedder() {
|
|
|
97453
97408
|
}
|
|
97454
97409
|
return _localEmbedder;
|
|
97455
97410
|
}
|
|
97456
|
-
async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch) {
|
|
97411
|
+
async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch, packing) {
|
|
97457
97412
|
if (embedderOverrides?.embedBatch) {
|
|
97458
|
-
return embedderOverrides.embedBatch(texts, embeddingConfig, signal, onSkip, onBatch);
|
|
97413
|
+
return embedderOverrides.embedBatch(texts, embeddingConfig, signal, onSkip, onBatch, packing);
|
|
97459
97414
|
}
|
|
97460
97415
|
if (texts.length === 0)
|
|
97461
97416
|
return [];
|
|
@@ -97465,7 +97420,7 @@ async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch) {
|
|
|
97465
97420
|
return embeddings;
|
|
97466
97421
|
}
|
|
97467
97422
|
if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
|
|
97468
|
-
return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal, onSkip, onBatch);
|
|
97423
|
+
return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal, onSkip, onBatch, packing);
|
|
97469
97424
|
}
|
|
97470
97425
|
const localModel = embeddingConfig?.localModel;
|
|
97471
97426
|
if (!localModel) {
|
|
@@ -97483,70 +97438,185 @@ async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch) {
|
|
|
97483
97438
|
return results;
|
|
97484
97439
|
}
|
|
97485
97440
|
|
|
97486
|
-
// src/
|
|
97487
|
-
|
|
97488
|
-
|
|
97489
|
-
|
|
97490
|
-
|
|
97491
|
-
|
|
97492
|
-
|
|
97493
|
-
|
|
97441
|
+
// src/llm/embedders/provider-limits.ts
|
|
97442
|
+
var DEFAULT_WINDOW_TOKENS = 8192;
|
|
97443
|
+
var CHARS_PER_TOKEN_TAIL = 2.6;
|
|
97444
|
+
var UNIT_HEADER_MARGIN_TOKENS = 64;
|
|
97445
|
+
var CALIBRATION_PERCENTILE = 0.01;
|
|
97446
|
+
var TOKENIZE_PRESENCE_PROBE_TEXT = "ping";
|
|
97447
|
+
var CALIBRATION_SHAPES = [
|
|
97448
|
+
"This is a short sentence describing typical prose content used to calibrate the tokenizer.",
|
|
97449
|
+
"```ts\nfunction add(a: number, b: number): number {\n return a + b;\n}\n```",
|
|
97450
|
+
`| Column A | Column B | Column C |
|
|
97451
|
+
| --- | --- | --- |
|
|
97452
|
+
| 1 | 2 | 3 |
|
|
97453
|
+
| 4 | 5 | 6 |`,
|
|
97454
|
+
`- first item in a list
|
|
97455
|
+
- second item in a list
|
|
97456
|
+
- third item, a little longer than the rest`,
|
|
97457
|
+
"See [the reference documentation](https://example.com/docs/reference) for more detail on this API.",
|
|
97458
|
+
"日本語のテキストは英語と比べてトークンあたりの文字数が大きく異なることがあります。",
|
|
97459
|
+
"SELECT id, name, description FROM entries WHERE tags LIKE '%embedding%' ORDER BY updated_at DESC LIMIT 50;",
|
|
97460
|
+
"A longer paragraph mixing punctuation, numbers (like 42 and 3.14), and technical terms such as `tokenizer`, `embedding`, and `context window`."
|
|
97461
|
+
];
|
|
97462
|
+
function percentileIndex(length, percentile) {
|
|
97463
|
+
if (length <= 1)
|
|
97464
|
+
return 0;
|
|
97465
|
+
return Math.max(0, Math.min(length - 1, Math.floor((length - 1) * percentile)));
|
|
97494
97466
|
}
|
|
97495
|
-
|
|
97496
|
-
|
|
97497
|
-
|
|
97498
|
-
|
|
97499
|
-
|
|
97500
|
-
|
|
97501
|
-
|
|
97502
|
-
|
|
97503
|
-
|
|
97504
|
-
|
|
97467
|
+
async function calibrateCharsPerToken(countTokens) {
|
|
97468
|
+
const ratios = [];
|
|
97469
|
+
for (const text of CALIBRATION_SHAPES) {
|
|
97470
|
+
try {
|
|
97471
|
+
const tokens = await countTokens(text);
|
|
97472
|
+
if (tokens > 0)
|
|
97473
|
+
ratios.push(text.length / tokens);
|
|
97474
|
+
} catch {}
|
|
97475
|
+
}
|
|
97476
|
+
if (ratios.length === 0)
|
|
97477
|
+
return CHARS_PER_TOKEN_TAIL;
|
|
97478
|
+
ratios.sort((a, b) => a - b);
|
|
97479
|
+
return ratios[percentileIndex(ratios.length, CALIBRATION_PERCENTILE)];
|
|
97505
97480
|
}
|
|
97506
|
-
function
|
|
97507
|
-
|
|
97508
|
-
|
|
97509
|
-
|
|
97510
|
-
contended.cause = error2;
|
|
97511
|
-
return contended;
|
|
97481
|
+
function timedFetch(fetchImpl, url2, init, timeoutMs, externalSignal) {
|
|
97482
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
97483
|
+
const signal = externalSignal ? AbortSignal.any([externalSignal, timeoutSignal]) : timeoutSignal;
|
|
97484
|
+
return fetchImpl(url2, { ...init, signal });
|
|
97512
97485
|
}
|
|
97513
|
-
|
|
97514
|
-
|
|
97515
|
-
|
|
97516
|
-
|
|
97517
|
-
return
|
|
97486
|
+
async function readJson(res) {
|
|
97487
|
+
try {
|
|
97488
|
+
return await res.json();
|
|
97489
|
+
} catch {
|
|
97490
|
+
return;
|
|
97518
97491
|
}
|
|
97519
|
-
|
|
97520
|
-
|
|
97492
|
+
}
|
|
97493
|
+
async function probeLlamaCppCountTokens(origin, fetchImpl, timeoutMs, probeSignal) {
|
|
97494
|
+
const tokenizeOnce = (text, signal) => timedFetch(fetchImpl, `${origin}/tokenize`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ content: text }) }, timeoutMs, signal);
|
|
97495
|
+
const presence = await tokenizeOnce(TOKENIZE_PRESENCE_PROBE_TEXT, probeSignal);
|
|
97496
|
+
if (!presence.ok)
|
|
97497
|
+
return;
|
|
97498
|
+
const presenceBody = await readJson(presence);
|
|
97499
|
+
if (!Array.isArray(presenceBody?.tokens))
|
|
97500
|
+
return;
|
|
97501
|
+
return async (text) => {
|
|
97502
|
+
const res = await tokenizeOnce(text, undefined);
|
|
97503
|
+
if (!res.ok)
|
|
97504
|
+
throw new Error(`llama.cpp /tokenize request failed (${res.status})`);
|
|
97505
|
+
const json = await readJson(res);
|
|
97506
|
+
if (!Array.isArray(json?.tokens))
|
|
97507
|
+
throw new Error("Unexpected /tokenize response: missing tokens array");
|
|
97508
|
+
return json.tokens.length;
|
|
97509
|
+
};
|
|
97510
|
+
}
|
|
97511
|
+
async function probeLlamaCpp(origin, config, fetchImpl, timeoutMs, signal) {
|
|
97512
|
+
const res = await timedFetch(fetchImpl, `${origin}/props`, { method: "GET" }, timeoutMs, signal);
|
|
97513
|
+
if (!res.ok)
|
|
97514
|
+
return;
|
|
97515
|
+
const body = await readJson(res);
|
|
97516
|
+
const windowTokens = body?.default_generation_settings?.n_ctx;
|
|
97517
|
+
if (typeof windowTokens !== "number" || !Number.isFinite(windowTokens) || windowTokens <= 0)
|
|
97518
|
+
return;
|
|
97519
|
+
const probedSlots = typeof body?.total_slots === "number" && body.total_slots > 0 ? body.total_slots : 1;
|
|
97520
|
+
const slots = config.concurrency ?? probedSlots;
|
|
97521
|
+
const countTokens = await probeLlamaCppCountTokens(origin, fetchImpl, timeoutMs, signal).catch(() => {
|
|
97522
|
+
return;
|
|
97523
|
+
});
|
|
97524
|
+
const charsPerToken = countTokens ? await calibrateCharsPerToken(countTokens) : CHARS_PER_TOKEN_TAIL;
|
|
97525
|
+
return { windowTokens, slots, source: "llama.cpp", charsPerToken };
|
|
97526
|
+
}
|
|
97527
|
+
var OLLAMA_DEFAULT_SLOTS = 1;
|
|
97528
|
+
function findOllamaContextLength(modelInfo) {
|
|
97529
|
+
for (const [key, value] of Object.entries(modelInfo)) {
|
|
97530
|
+
if (key.endsWith(".context_length") && typeof value === "number" && Number.isFinite(value) && value > 0) {
|
|
97531
|
+
return value;
|
|
97532
|
+
}
|
|
97521
97533
|
}
|
|
97522
|
-
return
|
|
97534
|
+
return;
|
|
97523
97535
|
}
|
|
97524
|
-
function
|
|
97525
|
-
|
|
97536
|
+
async function probeOllama(origin, config, fetchImpl, timeoutMs, signal) {
|
|
97537
|
+
if (!config.model)
|
|
97538
|
+
return;
|
|
97539
|
+
const res = await timedFetch(fetchImpl, `${origin}/api/show`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: config.model }) }, timeoutMs, signal);
|
|
97540
|
+
if (!res.ok)
|
|
97541
|
+
return;
|
|
97542
|
+
const body = await readJson(res);
|
|
97543
|
+
if (!body?.model_info)
|
|
97544
|
+
return;
|
|
97545
|
+
const windowTokens = findOllamaContextLength(body.model_info);
|
|
97546
|
+
if (windowTokens === undefined)
|
|
97547
|
+
return;
|
|
97548
|
+
return {
|
|
97549
|
+
windowTokens,
|
|
97550
|
+
slots: config.concurrency ?? OLLAMA_DEFAULT_SLOTS,
|
|
97551
|
+
source: "ollama",
|
|
97552
|
+
charsPerToken: CHARS_PER_TOKEN_TAIL
|
|
97553
|
+
};
|
|
97526
97554
|
}
|
|
97527
|
-
|
|
97528
|
-
|
|
97529
|
-
|
|
97530
|
-
|
|
97531
|
-
|
|
97532
|
-
|
|
97533
|
-
|
|
97534
|
-
|
|
97535
|
-
|
|
97555
|
+
function resolveOrigin(endpoint) {
|
|
97556
|
+
if (!endpoint)
|
|
97557
|
+
return;
|
|
97558
|
+
try {
|
|
97559
|
+
const parsed = new URL(endpoint);
|
|
97560
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
|
|
97561
|
+
return;
|
|
97562
|
+
return parsed.origin;
|
|
97563
|
+
} catch {
|
|
97564
|
+
return;
|
|
97536
97565
|
}
|
|
97537
|
-
|
|
97538
|
-
|
|
97566
|
+
}
|
|
97567
|
+
function resolveProbeTimeoutMs(config) {
|
|
97568
|
+
return config.timeoutMs ?? HEALTH_PROBE_TIMEOUT_MS;
|
|
97569
|
+
}
|
|
97570
|
+
function defaultLimits(config) {
|
|
97539
97571
|
return {
|
|
97540
|
-
|
|
97541
|
-
|
|
97542
|
-
|
|
97572
|
+
windowTokens: DEFAULT_WINDOW_TOKENS,
|
|
97573
|
+
slots: config.concurrency ?? 1,
|
|
97574
|
+
source: "default",
|
|
97575
|
+
charsPerToken: CHARS_PER_TOKEN_TAIL
|
|
97543
97576
|
};
|
|
97544
97577
|
}
|
|
97545
|
-
function
|
|
97546
|
-
|
|
97547
|
-
const mid = Math.floor(sorted.length / 2);
|
|
97548
|
-
return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
|
|
97578
|
+
function isUsableWindow(limits) {
|
|
97579
|
+
return unitMaxChars(limits) > 0;
|
|
97549
97580
|
}
|
|
97581
|
+
async function probeProviderLimitsUncached(config, opts) {
|
|
97582
|
+
const origin = resolveOrigin(config.endpoint);
|
|
97583
|
+
if (!origin)
|
|
97584
|
+
return defaultLimits(config);
|
|
97585
|
+
const fetchImpl = opts?.fetch ?? fetch;
|
|
97586
|
+
const timeoutMs = resolveProbeTimeoutMs(config);
|
|
97587
|
+
const signal = opts?.signal;
|
|
97588
|
+
const llamaCpp = await probeLlamaCpp(origin, config, fetchImpl, timeoutMs, signal).catch(() => {
|
|
97589
|
+
return;
|
|
97590
|
+
});
|
|
97591
|
+
if (llamaCpp && isUsableWindow(llamaCpp))
|
|
97592
|
+
return llamaCpp;
|
|
97593
|
+
const ollama = await probeOllama(origin, config, fetchImpl, timeoutMs, signal).catch(() => {
|
|
97594
|
+
return;
|
|
97595
|
+
});
|
|
97596
|
+
if (ollama && isUsableWindow(ollama))
|
|
97597
|
+
return ollama;
|
|
97598
|
+
return defaultLimits(config);
|
|
97599
|
+
}
|
|
97600
|
+
var providerLimitsCache = new Map;
|
|
97601
|
+
async function probeProviderLimits(config, opts) {
|
|
97602
|
+
const cacheKey = JSON.stringify({
|
|
97603
|
+
endpoint: config.endpoint,
|
|
97604
|
+
model: config.model,
|
|
97605
|
+
concurrency: config.concurrency,
|
|
97606
|
+
timeoutMs: config.timeoutMs
|
|
97607
|
+
});
|
|
97608
|
+
const cached = providerLimitsCache.get(cacheKey);
|
|
97609
|
+
if (cached)
|
|
97610
|
+
return cached;
|
|
97611
|
+
const probe = probeProviderLimitsUncached(config, opts);
|
|
97612
|
+
providerLimitsCache.set(cacheKey, probe);
|
|
97613
|
+
return probe;
|
|
97614
|
+
}
|
|
97615
|
+
function unitMaxChars(limits) {
|
|
97616
|
+
return Math.max(0, Math.floor((limits.windowTokens - UNIT_HEADER_MARGIN_TOKENS) * limits.charsPerToken));
|
|
97617
|
+
}
|
|
97618
|
+
|
|
97619
|
+
// src/indexer/embedding-identity.ts
|
|
97550
97620
|
function deriveObservedEmbeddingIdentity(embedding, observedModel, observedVectorLen) {
|
|
97551
97621
|
if (isDeterministicEmbedEnabled()) {
|
|
97552
97622
|
return `deterministic:${DETERMINISTIC_EMBED_MODEL_ID}`;
|
|
@@ -97558,600 +97628,315 @@ function deriveObservedEmbeddingIdentity(embedding, observedModel, observedVecto
|
|
|
97558
97628
|
}
|
|
97559
97629
|
return `local:${embedding?.localModel ?? DEFAULT_LOCAL_MODEL}|${observedVectorLen}`;
|
|
97560
97630
|
}
|
|
97561
|
-
|
|
97562
|
-
|
|
97563
|
-
|
|
97564
|
-
|
|
97565
|
-
|
|
97566
|
-
|
|
97567
|
-
|
|
97568
|
-
|
|
97569
|
-
|
|
97570
|
-
|
|
97571
|
-
|
|
97572
|
-
observedModel = model;
|
|
97573
|
-
});
|
|
97574
|
-
} catch (error2) {
|
|
97575
|
-
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
97576
|
-
return {
|
|
97577
|
-
outcome: "unverifiable",
|
|
97578
|
-
message: `could not verify embedding compatibility (${message}); keeping existing vectors — rerun akm index when the endpoint is reachable`
|
|
97579
|
-
};
|
|
97580
|
-
}
|
|
97581
|
-
const observedVectorLen = canaryVectors.find((vector) => vector !== undefined)?.length;
|
|
97582
|
-
const observedIdentity = deriveObservedEmbeddingIdentity(config.embedding, observedModel, observedVectorLen);
|
|
97583
|
-
const storedIdentity = getMeta(db, "embeddingIdentity");
|
|
97584
|
-
if (storedIdentity && observedIdentity && storedIdentity === observedIdentity) {
|
|
97585
|
-
return { outcome: "keep", verified: true, identity: observedIdentity, viaIdentityMatch: true };
|
|
97586
|
-
}
|
|
97587
|
-
const pairs = samples.map((sample, i) => ({ stored: sample.vector, fresh: canaryVectors[i] }));
|
|
97588
|
-
const decision = decideEmbeddingCompatibility(pairs);
|
|
97589
|
-
if (decision.outcome === "unverifiable") {
|
|
97590
|
-
const message = skips[0]?.message ?? "embedding provider returned no vectors for the canary sample";
|
|
97591
|
-
return {
|
|
97592
|
-
outcome: "unverifiable",
|
|
97593
|
-
message: `could not verify embedding compatibility (${message}); keeping existing vectors — rerun akm index when the endpoint is reachable`
|
|
97594
|
-
};
|
|
97595
|
-
}
|
|
97596
|
-
if (decision.outcome === "keep") {
|
|
97597
|
-
return {
|
|
97598
|
-
outcome: "keep",
|
|
97599
|
-
verified: true,
|
|
97600
|
-
identity: observedIdentity,
|
|
97601
|
-
viaIdentityMatch: false,
|
|
97602
|
-
medianSimilarity: decision.medianSimilarity
|
|
97603
|
-
};
|
|
97604
|
-
}
|
|
97605
|
-
return {
|
|
97606
|
-
outcome: "rebuild",
|
|
97607
|
-
identity: observedIdentity,
|
|
97608
|
-
reason: `vectors differ (median similarity ${decision.medianSimilarity?.toFixed(3)})`
|
|
97609
|
-
};
|
|
97631
|
+
|
|
97632
|
+
// src/indexer/drain.ts
|
|
97633
|
+
var CIRCUIT_BREAKER_THRESHOLD = 3;
|
|
97634
|
+
var CIRCUIT_BREAKER_WINDOW = CIRCUIT_BREAKER_THRESHOLD * 2;
|
|
97635
|
+
function pushBreakerOutcome(window2, isFailure) {
|
|
97636
|
+
window2.push(isFailure);
|
|
97637
|
+
if (window2.length > CIRCUIT_BREAKER_WINDOW)
|
|
97638
|
+
window2.shift();
|
|
97639
|
+
}
|
|
97640
|
+
function breakerFailureCount(window2) {
|
|
97641
|
+
return window2.reduce((n, isFailure) => n + (isFailure ? 1 : 0), 0);
|
|
97610
97642
|
}
|
|
97643
|
+
var DRAIN_BATCH_PROGRESS_PREFIX = "[drain] batch ";
|
|
97611
97644
|
function throwIfAborted(signal) {
|
|
97612
97645
|
if (signal?.aborted) {
|
|
97613
|
-
throw signal.reason instanceof Error ? signal.reason : new Error("
|
|
97646
|
+
throw signal.reason instanceof Error ? signal.reason : new Error("drain interrupted");
|
|
97614
97647
|
}
|
|
97615
97648
|
}
|
|
97616
|
-
|
|
97617
|
-
|
|
97618
|
-
|
|
97649
|
+
function selectAllUnitHashes(db) {
|
|
97650
|
+
return db.prepare("SELECT unit_hash FROM unit_texts ORDER BY unit_hash").all().map((row) => row.unit_hash);
|
|
97651
|
+
}
|
|
97652
|
+
function fetchUnitTexts(db, hashes) {
|
|
97653
|
+
const texts = new Map;
|
|
97654
|
+
for (let offset = 0;offset < hashes.length; offset += SQLITE_CHUNK_SIZE) {
|
|
97655
|
+
const chunk2 = hashes.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
97656
|
+
const placeholders = chunk2.map(() => "?").join(",");
|
|
97657
|
+
const rows = db.prepare(`SELECT unit_hash, text FROM unit_texts WHERE unit_hash IN (${placeholders})`).all(...chunk2);
|
|
97658
|
+
for (const row of rows)
|
|
97659
|
+
texts.set(row.unit_hash, row.text);
|
|
97619
97660
|
}
|
|
97620
|
-
|
|
97661
|
+
return texts;
|
|
97662
|
+
}
|
|
97663
|
+
async function resolveEmbeddingPacking(config, signal) {
|
|
97664
|
+
const base3 = config.embedding ?? {};
|
|
97665
|
+
const limits = await probeProviderLimits(base3, { signal });
|
|
97666
|
+
return {
|
|
97667
|
+
embeddingConfig: { ...base3, concurrency: base3.concurrency ?? limits.slots },
|
|
97668
|
+
packing: {
|
|
97669
|
+
tokenBudget: limits.windowTokens,
|
|
97670
|
+
charsPerToken: limits.charsPerToken,
|
|
97671
|
+
windowIsKnown: limits.source !== "default",
|
|
97672
|
+
ollamaNumCtx: limits.source === "ollama" ? limits.windowTokens : undefined
|
|
97673
|
+
}
|
|
97674
|
+
};
|
|
97675
|
+
}
|
|
97676
|
+
function emitCredentialDiagnostic(config, onProgress) {
|
|
97677
|
+
if (!onProgress || !hasRemoteEndpoint(config.embedding ?? {}))
|
|
97678
|
+
return;
|
|
97679
|
+
const endpoint = normalizeEmbeddingEndpoint(config.embedding?.endpoint ?? "");
|
|
97680
|
+
const credential = describeEmbeddingCredential(config.embedding?.apiKey);
|
|
97681
|
+
const configFileSuffix = isVerbose() ? `; config: ${getConfigPath()}` : "";
|
|
97682
|
+
onProgress(`[embed] endpoint ${endpoint}, model ${config.embedding?.model ?? "unknown"}; credential: ${credential}${configFileSuffix}`);
|
|
97683
|
+
}
|
|
97684
|
+
function formatDoneLine(counts) {
|
|
97685
|
+
return `[drain] done: ${counts.pending} pending, ${counts.embedded} embedded, ${counts.failed} failed, ` + `${counts.skipped} skipped (identity: ${counts.identity ?? "unknown"})`;
|
|
97686
|
+
}
|
|
97687
|
+
async function drainEmbeddingQueue(db, config, opts = {}) {
|
|
97688
|
+
throwIfAborted(opts.signal);
|
|
97689
|
+
let identity3 = getMeta(db, "embeddingIdentity") ?? null;
|
|
97621
97690
|
if (config.semanticSearchMode === "off") {
|
|
97622
|
-
|
|
97623
|
-
|
|
97624
|
-
|
|
97625
|
-
|
|
97626
|
-
|
|
97627
|
-
|
|
97628
|
-
|
|
97629
|
-
|
|
97630
|
-
|
|
97631
|
-
|
|
97632
|
-
|
|
97633
|
-
|
|
97634
|
-
|
|
97635
|
-
|
|
97636
|
-
|
|
97637
|
-
const
|
|
97638
|
-
|
|
97639
|
-
|
|
97640
|
-
const
|
|
97641
|
-
if (
|
|
97642
|
-
|
|
97643
|
-
|
|
97644
|
-
|
|
97645
|
-
|
|
97646
|
-
|
|
97647
|
-
|
|
97648
|
-
|
|
97649
|
-
|
|
97650
|
-
|
|
97651
|
-
|
|
97652
|
-
|
|
97653
|
-
|
|
97654
|
-
|
|
97655
|
-
|
|
97656
|
-
|
|
97657
|
-
|
|
97658
|
-
|
|
97659
|
-
|
|
97660
|
-
purgeEmbeddings(db, { dropVecTable: true });
|
|
97661
|
-
purgeEmbeddingSalvage(db);
|
|
97662
|
-
deleteMeta(db, "embeddingDim");
|
|
97663
|
-
setMeta(db, "embeddingFingerprint", currentFingerprint);
|
|
97664
|
-
if (decision.identity)
|
|
97665
|
-
setMeta(db, "embeddingIdentity", decision.identity);
|
|
97666
|
-
else
|
|
97667
|
-
deleteMeta(db, "embeddingIdentity");
|
|
97668
|
-
})();
|
|
97669
|
-
targetEntryIds = undefined;
|
|
97670
|
-
rebuildReason = decision.reason;
|
|
97671
|
-
} else {
|
|
97672
|
-
setMeta(db, "embeddingFingerprint", currentFingerprint);
|
|
97673
|
-
relabelEmbeddingSalvageFingerprint(db, storedFingerprint, currentFingerprint);
|
|
97674
|
-
if (decision.identity)
|
|
97675
|
-
setMeta(db, "embeddingIdentity", decision.identity);
|
|
97676
|
-
if (decision.verified) {
|
|
97677
|
-
const keptCount = getEmbeddingCount(db);
|
|
97678
|
-
const detail = decision.viaIdentityMatch ? "server-reported model unchanged" : `stored vectors are compatible (median similarity ${decision.medianSimilarity?.toFixed(3)})`;
|
|
97679
|
-
const message = `[embed] embedding model renamed (${storedFingerprint} → ${currentFingerprint}); ${detail}, keeping ${keptCount} embedding${keptCount === 1 ? "" : "s"}.`;
|
|
97680
|
-
warn(message);
|
|
97681
|
-
onProgress({ phase: "embeddings", message });
|
|
97682
|
-
}
|
|
97691
|
+
return { pending: 0, embedded: 0, failed: 0, skipped: 0, identity: identity3 };
|
|
97692
|
+
}
|
|
97693
|
+
const candidateHashes = opts.onlyHashes ? [...new Set(opts.onlyHashes)] : selectAllUnitHashes(db);
|
|
97694
|
+
const missingHashes = identity3 ? listMissingHashes(db, candidateHashes, identity3) : candidateHashes;
|
|
97695
|
+
const pending = missingHashes.length;
|
|
97696
|
+
const emitDone = (counts) => {
|
|
97697
|
+
opts.onProgress?.(formatDoneLine(counts));
|
|
97698
|
+
return counts;
|
|
97699
|
+
};
|
|
97700
|
+
if (!isVecAvailable(db)) {
|
|
97701
|
+
return emitDone({ pending, embedded: 0, failed: 0, skipped: pending, identity: identity3 });
|
|
97702
|
+
}
|
|
97703
|
+
if (pending === 0) {
|
|
97704
|
+
return emitDone({ pending: 0, embedded: 0, failed: 0, skipped: 0, identity: identity3 });
|
|
97705
|
+
}
|
|
97706
|
+
const boundedHashes = opts.limit !== undefined ? missingHashes.slice(0, opts.limit) : missingHashes;
|
|
97707
|
+
const textByHash = fetchUnitTexts(db, boundedHashes);
|
|
97708
|
+
const orderedHashes = boundedHashes.filter((hash4) => textByHash.has(hash4));
|
|
97709
|
+
const texts = orderedHashes.map((hash4) => textByHash.get(hash4));
|
|
97710
|
+
if (texts.length === 0) {
|
|
97711
|
+
return emitDone({ pending, embedded: 0, failed: 0, skipped: 0, identity: identity3 });
|
|
97712
|
+
}
|
|
97713
|
+
emitCredentialDiagnostic(config, opts.onProgress);
|
|
97714
|
+
const { embeddingConfig, packing } = await resolveEmbeddingPacking(config, opts.signal);
|
|
97715
|
+
let embedded = 0;
|
|
97716
|
+
let failed = 0;
|
|
97717
|
+
let batchNumber = 0;
|
|
97718
|
+
const singleDocFailureWindow = [];
|
|
97719
|
+
const networkErrorFailureWindow = [];
|
|
97720
|
+
let identityDecidedThisCall = false;
|
|
97721
|
+
const onSkip = (skip) => {
|
|
97722
|
+
failed++;
|
|
97723
|
+
if (!skip.batchStart)
|
|
97724
|
+
return;
|
|
97725
|
+
if (skip.reason === "context-window-exceeded") {
|
|
97726
|
+
singleDocFailureWindow.length = 0;
|
|
97727
|
+
networkErrorFailureWindow.length = 0;
|
|
97728
|
+
return;
|
|
97683
97729
|
}
|
|
97684
|
-
|
|
97685
|
-
|
|
97686
|
-
|
|
97687
|
-
|
|
97688
|
-
throwIfAborted(signal);
|
|
97689
|
-
const allEntries = getAllEntriesForEmbedding(db, targetEntryIds);
|
|
97690
|
-
let vecFailedCount = 0;
|
|
97691
|
-
let vecUnavailableCount = 0;
|
|
97692
|
-
const { reusedCount, remaining: candidateEntries } = reuseSalvagedEmbeddings(db, allEntries, currentFingerprint, (entry, embedding) => {
|
|
97693
|
-
const result = upsertEmbedding(db, entry.id, embedding);
|
|
97694
|
-
if (result.vec === "failed")
|
|
97695
|
-
vecFailedCount++;
|
|
97696
|
-
if (result.vec === "unavailable")
|
|
97697
|
-
vecUnavailableCount++;
|
|
97698
|
-
return result.stored;
|
|
97699
|
-
});
|
|
97700
|
-
if (reusedCount > 0) {
|
|
97701
|
-
onProgress({
|
|
97702
|
-
phase: "embeddings",
|
|
97703
|
-
message: `Reused ${reusedCount} embedding${reusedCount === 1 ? "" : "s"} from the previous generation; embedding ${candidateEntries.length} new.`
|
|
97704
|
-
});
|
|
97730
|
+
pushBreakerOutcome(singleDocFailureWindow, skip.batchSize === 1);
|
|
97731
|
+
pushBreakerOutcome(networkErrorFailureWindow, skip.failureKind === "network-error");
|
|
97732
|
+
if (breakerFailureCount(singleDocFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD || breakerFailureCount(networkErrorFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD) {
|
|
97733
|
+
return false;
|
|
97705
97734
|
}
|
|
97706
|
-
|
|
97707
|
-
|
|
97708
|
-
|
|
97709
|
-
|
|
97710
|
-
|
|
97711
|
-
|
|
97712
|
-
|
|
97713
|
-
|
|
97714
|
-
|
|
97715
|
-
}
|
|
97716
|
-
let truncatedCount = 0;
|
|
97717
|
-
const texts = [];
|
|
97718
|
-
const pendingEntries = [];
|
|
97719
|
-
for (const entry of candidateEntries) {
|
|
97720
|
-
const capped = capEmbeddingText(entry.searchText, maxInputTokens);
|
|
97721
|
-
if (capped.text.length === 0)
|
|
97735
|
+
return;
|
|
97736
|
+
};
|
|
97737
|
+
const onBatch = (indices, embeddings, model, outcome) => {
|
|
97738
|
+
if (outcome?.outcome === "retrying" || outcome?.outcome === "budget-lowered")
|
|
97739
|
+
return;
|
|
97740
|
+
const rows = [];
|
|
97741
|
+
for (let k2 = 0;k2 < indices.length; k2++) {
|
|
97742
|
+
const embedding = embeddings[k2];
|
|
97743
|
+
if (!embedding)
|
|
97722
97744
|
continue;
|
|
97723
|
-
|
|
97724
|
-
|
|
97725
|
-
|
|
97726
|
-
|
|
97727
|
-
|
|
97728
|
-
|
|
97729
|
-
|
|
97730
|
-
onProgress({ phase: "embeddings", message });
|
|
97731
|
-
}
|
|
97732
|
-
if (rebuildReason) {
|
|
97733
|
-
const message = `[embed] Re-embedding ${pendingEntries.length} entr${pendingEntries.length === 1 ? "y" : "ies"} because ${rebuildReason}`;
|
|
97734
|
-
onProgress({ phase: "embeddings", message });
|
|
97735
|
-
}
|
|
97736
|
-
onProgress({
|
|
97737
|
-
phase: "embeddings",
|
|
97738
|
-
message: `Generating embeddings for ${pendingEntries.length} entr${pendingEntries.length === 1 ? "y" : "ies"}.`
|
|
97739
|
-
});
|
|
97740
|
-
if (isVerbose()) {
|
|
97741
|
-
if (hasRemoteEndpoint(config.embedding ?? {})) {
|
|
97742
|
-
const tokenBudget = config.embedding?.maxTokens ?? DEFAULT_TOKEN_BUDGET;
|
|
97743
|
-
const maxCount = config.embedding?.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
|
|
97744
|
-
const batches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
|
|
97745
|
-
const batchNumberByIndex = new Map;
|
|
97746
|
-
batches.forEach((batch, batchIdx) => {
|
|
97747
|
-
for (const i of batch.indices)
|
|
97748
|
-
batchNumberByIndex.set(i, batchIdx + 1);
|
|
97749
|
-
});
|
|
97750
|
-
for (const [i, entry] of pendingEntries.entries()) {
|
|
97751
|
-
const chars = entry.searchText.length;
|
|
97752
|
-
const tokens = estimateTokenCount(entry.searchText);
|
|
97753
|
-
const batch = batches[batchNumberByIndex.get(i) - 1];
|
|
97754
|
-
const label = batch?.oversized ? "oversized (skipped)" : `batch ${batchNumberByIndex.get(i)}/${batches.length}`;
|
|
97755
|
-
warnVerbose(`[embed] ${entry.itemRef} (${chars} chars, est. ${tokens} tokens) → ${label}`);
|
|
97756
|
-
}
|
|
97757
|
-
} else {
|
|
97758
|
-
for (const entry of pendingEntries) {
|
|
97759
|
-
warnVerbose(`[embed] ${entry.itemRef} (${entry.searchText.length} chars, est. ${estimateTokenCount(entry.searchText)} tokens)`);
|
|
97745
|
+
const learned = deriveObservedEmbeddingIdentity(config.embedding, model, embedding.length);
|
|
97746
|
+
if (!identityDecidedThisCall) {
|
|
97747
|
+
identityDecidedThisCall = true;
|
|
97748
|
+
if (learned && learned !== identity3) {
|
|
97749
|
+
identity3 = learned;
|
|
97750
|
+
setMeta(db, "embeddingIdentity", identity3);
|
|
97751
|
+
dropOtherIdentities(db, identity3, embedding.length);
|
|
97760
97752
|
}
|
|
97761
97753
|
}
|
|
97754
|
+
const currentIdentity = identity3;
|
|
97755
|
+
if (currentIdentity === null || learned !== currentIdentity) {
|
|
97756
|
+
continue;
|
|
97757
|
+
}
|
|
97758
|
+
const hash4 = orderedHashes[indices[k2]];
|
|
97759
|
+
if (hash4)
|
|
97760
|
+
rows.push({ hash: hash4, identity: currentIdentity, vector: embedding });
|
|
97762
97761
|
}
|
|
97763
|
-
let
|
|
97764
|
-
|
|
97765
|
-
|
|
97766
|
-
|
|
97767
|
-
|
|
97768
|
-
|
|
97769
|
-
|
|
97770
|
-
|
|
97771
|
-
|
|
97772
|
-
|
|
97773
|
-
});
|
|
97774
|
-
}, 15000);
|
|
97775
|
-
const skips = [];
|
|
97776
|
-
const embedStart = Date.now();
|
|
97777
|
-
let consecutiveSingleDocFailures = 0;
|
|
97778
|
-
let consecutiveNetworkErrorFailures = 0;
|
|
97779
|
-
let circuitBreakerReason;
|
|
97780
|
-
const onSkip = (skip) => {
|
|
97781
|
-
skips.push(skip);
|
|
97782
|
-
if (!skip.batchStart)
|
|
97783
|
-
return;
|
|
97784
|
-
if (skip.reason === "context-window-exceeded") {
|
|
97785
|
-
consecutiveSingleDocFailures = 0;
|
|
97786
|
-
consecutiveNetworkErrorFailures = 0;
|
|
97787
|
-
return;
|
|
97788
|
-
}
|
|
97789
|
-
consecutiveSingleDocFailures = skip.batchSize === 1 ? consecutiveSingleDocFailures + 1 : 0;
|
|
97790
|
-
consecutiveNetworkErrorFailures = skip.failureKind === "network-error" ? consecutiveNetworkErrorFailures + 1 : 0;
|
|
97791
|
-
if (consecutiveSingleDocFailures >= CIRCUIT_BREAKER_THRESHOLD || consecutiveNetworkErrorFailures >= CIRCUIT_BREAKER_THRESHOLD) {
|
|
97792
|
-
circuitBreakerReason = skip.message;
|
|
97793
|
-
return false;
|
|
97794
|
-
}
|
|
97795
|
-
return;
|
|
97796
|
-
};
|
|
97797
|
-
let observedModel;
|
|
97798
|
-
let observedVectorLen;
|
|
97799
|
-
const reportPerBatchLine = hasRemoteEndpoint(config.embedding ?? {});
|
|
97800
|
-
const onBatch = (indices, batchEmbeddings, model, outcome) => {
|
|
97801
|
-
if (outcome?.outcome === "retrying") {
|
|
97802
|
-
if (reportPerBatchLine) {
|
|
97803
|
-
onProgress({
|
|
97804
|
-
phase: "embeddings",
|
|
97805
|
-
message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens → retrying after ${(outcome.elapsedMs / 1000).toFixed(1)} s`
|
|
97806
|
-
});
|
|
97807
|
-
}
|
|
97808
|
-
return;
|
|
97809
|
-
}
|
|
97810
|
-
if (outcome?.outcome === "budget-lowered") {
|
|
97811
|
-
if (reportPerBatchLine) {
|
|
97812
|
-
onProgress({
|
|
97813
|
-
phase: "embeddings",
|
|
97814
|
-
message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens → ${outcome.reason}`
|
|
97815
|
-
});
|
|
97816
|
-
}
|
|
97817
|
-
return;
|
|
97818
|
-
}
|
|
97819
|
-
if (model)
|
|
97820
|
-
observedModel = model;
|
|
97821
|
-
if (batchEmbeddings.some((embedding) => embedding !== undefined)) {
|
|
97822
|
-
consecutiveSingleDocFailures = 0;
|
|
97823
|
-
consecutiveNetworkErrorFailures = 0;
|
|
97824
|
-
}
|
|
97825
|
-
db.transaction(() => {
|
|
97826
|
-
for (let k2 = 0;k2 < indices.length; k2++) {
|
|
97827
|
-
const index = indices[k2];
|
|
97828
|
-
const entry = pendingEntries[index];
|
|
97829
|
-
if (!entry)
|
|
97830
|
-
continue;
|
|
97831
|
-
const embedding = batchEmbeddings[k2];
|
|
97832
|
-
if (!embedding) {
|
|
97833
|
-
embedFailedCount++;
|
|
97834
|
-
continue;
|
|
97835
|
-
}
|
|
97836
|
-
if (observedVectorLen === undefined)
|
|
97837
|
-
observedVectorLen = embedding.length;
|
|
97838
|
-
const result = upsertEmbedding(db, entry.id, embedding);
|
|
97839
|
-
if (result.stored) {
|
|
97840
|
-
storedCount++;
|
|
97841
|
-
storedTokens += estimateTokenCount(texts[index]);
|
|
97842
|
-
} else {
|
|
97843
|
-
skippedCount++;
|
|
97844
|
-
}
|
|
97845
|
-
if (result.vec === "failed")
|
|
97846
|
-
vecFailedCount++;
|
|
97847
|
-
if (result.vec === "unavailable")
|
|
97848
|
-
vecUnavailableCount++;
|
|
97849
|
-
}
|
|
97850
|
-
})();
|
|
97851
|
-
if (outcome && outcome.reason !== "oversized" && reportPerBatchLine) {
|
|
97852
|
-
const elapsedSeconds2 = (outcome.elapsedMs / 1000).toFixed(1);
|
|
97853
|
-
const outcomeLabel = outcome.outcome === "stored" ? `${outcome.docCount} stored (${elapsedSeconds2} s)` : `failed: ${outcome.reason}`;
|
|
97854
|
-
onProgress({
|
|
97855
|
-
phase: "embeddings",
|
|
97856
|
-
message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens → ${outcomeLabel}`
|
|
97857
|
-
});
|
|
97762
|
+
let storageBreakerTripped = false;
|
|
97763
|
+
if (rows.length > 0) {
|
|
97764
|
+
const result = upsertUnitVectors(db, rows);
|
|
97765
|
+
embedded += result.inserted;
|
|
97766
|
+
failed += result.failed;
|
|
97767
|
+
if (result.failed > 0) {
|
|
97768
|
+
pushBreakerOutcome(singleDocFailureWindow, true);
|
|
97769
|
+
pushBreakerOutcome(networkErrorFailureWindow, true);
|
|
97770
|
+
if (breakerFailureCount(singleDocFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD || breakerFailureCount(networkErrorFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD) {
|
|
97771
|
+
storageBreakerTripped = true;
|
|
97858
97772
|
}
|
|
97859
|
-
onProgress({
|
|
97860
|
-
phase: "embeddings",
|
|
97861
|
-
message: `Embedded ${storedCount}/${pendingEntries.length} entries.`
|
|
97862
|
-
});
|
|
97863
|
-
};
|
|
97864
|
-
await embedBatch(texts, config.embedding, signal, onSkip, onBatch);
|
|
97865
|
-
throwIfAborted(signal);
|
|
97866
|
-
const elapsedSeconds = Math.max((Date.now() - embedStart) / 1000, 0.001);
|
|
97867
|
-
if (skippedCount > 0) {
|
|
97868
|
-
warn(`[embed] ${skippedCount} embedding${skippedCount === 1 ? "" : "s"} skipped (entry deleted between queue and write)`);
|
|
97869
|
-
}
|
|
97870
|
-
const vecGenerationComplete = targetEntryIds === undefined ? isVecFastPathComplete(db) : vecFastPathWasReady;
|
|
97871
|
-
setVecFastPathReady(db, vecFailedCount === 0 && vecUnavailableCount === 0 && vecGenerationComplete);
|
|
97872
|
-
if (vecFailedCount > 0) {
|
|
97873
|
-
warn(`[embed] ${vecFailedCount} sqlite-vec fast-path insert${vecFailedCount === 1 ? "" : "s"} failed — ` + "semantic search will use the slower JS-cosine fallback over stored embeddings. " + "Rebuild with 'akm index --full' after resolving the vec table (often a vector-dimension mismatch).");
|
|
97874
|
-
}
|
|
97875
|
-
const entriesPerSec = storedCount / elapsedSeconds;
|
|
97876
|
-
const tokensPerSec = storedTokens / elapsedSeconds;
|
|
97877
|
-
const totalStored = storedCount + reusedCount;
|
|
97878
|
-
const oversizedSkips = skips.filter((skip) => skip.reason === "context-window-exceeded");
|
|
97879
|
-
const timedOutSkips = skips.filter((skip) => skip.reason === "batch-request-failed" && skip.failureKind === "timeout");
|
|
97880
|
-
const failedSkips = skips.filter((skip) => skip.reason === "batch-request-failed" && skip.failureKind !== "timeout");
|
|
97881
|
-
const throughputLine = reusedCount > 0 ? `Stored ${totalStored} embedding${totalStored === 1 ? "" : "s"} (${reusedCount} reused, ${storedCount} newly embedded) in ${elapsedSeconds.toFixed(1)}s (${entriesPerSec.toFixed(1)} entries/s, ~${Math.round(tokensPerSec)} tokens/s)` : `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"} in ${elapsedSeconds.toFixed(1)}s (${entriesPerSec.toFixed(1)} entries/s, ~${Math.round(tokensPerSec)} tokens/s)`;
|
|
97882
|
-
onProgress({
|
|
97883
|
-
phase: "embeddings",
|
|
97884
|
-
message: `${throughputLine}; ${oversizedSkips.length} oversized skipped, ${timedOutSkips.length} timed out, ${failedSkips.length} failed.`
|
|
97885
|
-
});
|
|
97886
|
-
const printSkipList = (label, skipList) => {
|
|
97887
|
-
if (skipList.length === 0)
|
|
97888
|
-
return;
|
|
97889
|
-
const limit = isVerbose() ? skipList.length : 20;
|
|
97890
|
-
const listed = skipList.slice(0, limit).map((skip) => ` - ${pendingEntries[skip.index]?.itemRef ?? skip.index}: ${skip.message}`).join(`
|
|
97891
|
-
`);
|
|
97892
|
-
const more = skipList.length > limit ? `
|
|
97893
|
-
...and ${skipList.length - limit} more` : "";
|
|
97894
|
-
onProgress({ phase: "embeddings", message: `[embed] ${label} skipped:
|
|
97895
|
-
${listed}${more}` });
|
|
97896
|
-
};
|
|
97897
|
-
printSkipList("oversized documents", oversizedSkips);
|
|
97898
|
-
printSkipList("timed-out documents", timedOutSkips);
|
|
97899
|
-
printSkipList("failed documents", failedSkips);
|
|
97900
|
-
setMeta(db, "embeddingFingerprint", currentFingerprint);
|
|
97901
|
-
const observedIdentity = deriveObservedEmbeddingIdentity(config.embedding, observedModel, observedVectorLen);
|
|
97902
|
-
if (observedIdentity)
|
|
97903
|
-
setMeta(db, "embeddingIdentity", observedIdentity);
|
|
97904
|
-
if (circuitBreakerReason !== undefined) {
|
|
97905
|
-
const message = `embedding provider failed ${CIRCUIT_BREAKER_THRESHOLD} consecutive batches ` + `(last: ${circuitBreakerReason}); stopped after ${storedCount} embedding${storedCount === 1 ? "" : "s"} ` + "were stored — rerun akm index when the endpoint is healthy";
|
|
97906
|
-
warn(`[embed] ${message}`);
|
|
97907
|
-
onProgress({ phase: "embeddings", message });
|
|
97908
|
-
return { success: false, message, vecInsertFailures: vecFailedCount };
|
|
97909
|
-
}
|
|
97910
|
-
if (storedCount === 0 && embedFailedCount > 0) {
|
|
97911
|
-
const firstMessage = skips[0]?.message ?? "All embeddings failed.";
|
|
97912
|
-
return {
|
|
97913
|
-
success: false,
|
|
97914
|
-
message: `All ${embedFailedCount} embedding batch(es) failed: ${firstMessage}`
|
|
97915
|
-
};
|
|
97916
97773
|
}
|
|
97917
|
-
purgeEmbeddingSalvage(db);
|
|
97918
|
-
return { success: true, vecInsertFailures: vecFailedCount };
|
|
97919
|
-
} finally {
|
|
97920
|
-
if (heartbeatTimer)
|
|
97921
|
-
clearInterval(heartbeatTimer);
|
|
97922
97774
|
}
|
|
97923
|
-
|
|
97924
|
-
|
|
97925
|
-
|
|
97926
|
-
|
|
97927
|
-
|
|
97928
|
-
|
|
97929
|
-
|
|
97930
|
-
|
|
97931
|
-
|
|
97932
|
-
|
|
97933
|
-
|
|
97934
|
-
|
|
97935
|
-
|
|
97936
|
-
|
|
97937
|
-
|
|
97938
|
-
|
|
97939
|
-
const
|
|
97940
|
-
const
|
|
97941
|
-
|
|
97942
|
-
setMeta(db, "hasEmbeddings", ready ? "1" : "0");
|
|
97775
|
+
if (embeddings.some((embedding) => embedding !== undefined)) {
|
|
97776
|
+
pushBreakerOutcome(singleDocFailureWindow, false);
|
|
97777
|
+
pushBreakerOutcome(networkErrorFailureWindow, false);
|
|
97778
|
+
}
|
|
97779
|
+
batchNumber++;
|
|
97780
|
+
if (opts.onProgress) {
|
|
97781
|
+
const docCount = outcome?.docCount ?? indices.length;
|
|
97782
|
+
const label = outcome && outcome.outcome !== "stored" ? `failed: ${outcome.reason ?? "unknown"}` : `${rows.length} stored`;
|
|
97783
|
+
opts.onProgress(`${DRAIN_BATCH_PROGRESS_PREFIX}${batchNumber}: ${docCount} docs → ${label}`);
|
|
97784
|
+
}
|
|
97785
|
+
if (storageBreakerTripped) {
|
|
97786
|
+
throw new Error(`Circuit breaker: ${CIRCUIT_BREAKER_THRESHOLD} storage write failures while embedding; stopping further provider requests this call.`);
|
|
97787
|
+
}
|
|
97788
|
+
};
|
|
97789
|
+
await embedBatch(texts, embeddingConfig, opts.signal, onSkip, onBatch, packing);
|
|
97790
|
+
throwIfAborted(opts.signal);
|
|
97791
|
+
const attempted = texts.length;
|
|
97792
|
+
const skipped = Math.max(0, attempted - embedded - failed);
|
|
97793
|
+
return emitDone({ pending, embedded, failed, skipped, identity: identity3 });
|
|
97943
97794
|
}
|
|
97944
97795
|
|
|
97945
|
-
// src/indexer/
|
|
97946
|
-
|
|
97947
|
-
|
|
97948
|
-
// src/indexer/scan/drain-dir.ts
|
|
97949
|
-
import path63 from "node:path";
|
|
97796
|
+
// src/indexer/reconcile.ts
|
|
97797
|
+
import fs51 from "node:fs";
|
|
97798
|
+
import path62 from "node:path";
|
|
97950
97799
|
init_common();
|
|
97951
97800
|
init_recognition_util();
|
|
97801
|
+
init_warn();
|
|
97952
97802
|
|
|
97953
97803
|
// src/workflows/source-files.ts
|
|
97954
97804
|
init_common();
|
|
97955
97805
|
init_errors();
|
|
97956
97806
|
init_recognition_util();
|
|
97957
97807
|
init_warn();
|
|
97958
|
-
import fs49 from "node:fs";
|
|
97959
|
-
import path60 from "node:path";
|
|
97960
97808
|
|
|
97961
|
-
|
|
97962
|
-
|
|
97963
|
-
|
|
97964
|
-
super(message, code);
|
|
97965
|
-
this.sourcePaths = [...sourcePaths].sort(compareCodePoints);
|
|
97966
|
-
Object.setPrototypeOf(this, new.target.prototype);
|
|
97967
|
-
}
|
|
97968
|
-
}
|
|
97969
|
-
class WorkflowSourceLinkIdentityError extends WorkflowSourceRejectionError {
|
|
97970
|
-
constructor(sourcePath, targetPath) {
|
|
97971
|
-
super(`Workflow source ${sourcePath} resolves through a symlink to ${targetPath} with a different source format. ` + "The authored workflow path and resolved source must use the same .md or .yml format.", "WORKFLOW_SOURCE_INVALID", [sourcePath]);
|
|
97972
|
-
this.name = "WorkflowSourceLinkIdentityError";
|
|
97973
|
-
Object.setPrototypeOf(this, new.target.prototype);
|
|
97974
|
-
}
|
|
97975
|
-
}
|
|
97809
|
+
// src/indexer/enrich.ts
|
|
97810
|
+
init_errors();
|
|
97811
|
+
init_warn();
|
|
97976
97812
|
|
|
97977
|
-
|
|
97978
|
-
|
|
97979
|
-
|
|
97980
|
-
|
|
97981
|
-
|
|
97982
|
-
this.name = "WorkflowSourceLinkResolutionError";
|
|
97983
|
-
Object.setPrototypeOf(this, new.target.prototype);
|
|
97984
|
-
}
|
|
97985
|
-
}
|
|
97813
|
+
// src/llm/index-passes.ts
|
|
97814
|
+
init_warn();
|
|
97815
|
+
var NO_LOWERING_NOTICES = Object.freeze([]);
|
|
97816
|
+
// src/llm/structured-call.ts
|
|
97817
|
+
init_errors();
|
|
97986
97818
|
|
|
97987
|
-
|
|
97988
|
-
|
|
97989
|
-
|
|
97990
|
-
|
|
97991
|
-
|
|
97992
|
-
|
|
97993
|
-
|
|
97994
|
-
|
|
97819
|
+
// src/indexer/enrich.ts
|
|
97820
|
+
init_metadata();
|
|
97821
|
+
|
|
97822
|
+
// src/indexer/units/unit.ts
|
|
97823
|
+
init_markdown();
|
|
97824
|
+
|
|
97825
|
+
// src/core/hash.ts
|
|
97826
|
+
import { createHash as createHash10 } from "node:crypto";
|
|
97827
|
+
function hashEmbeddableText(text) {
|
|
97828
|
+
return createHash10("sha256").update(text, "utf8").digest("hex");
|
|
97995
97829
|
}
|
|
97996
|
-
|
|
97997
|
-
|
|
97998
|
-
|
|
97999
|
-
|
|
98000
|
-
|
|
98001
|
-
|
|
98002
|
-
|
|
98003
|
-
|
|
98004
|
-
|
|
98005
|
-
const extension = path60.posix.extname(ownedPath);
|
|
98006
|
-
if (!WORKFLOW_EXTENSIONS.includes(extension.toLowerCase()))
|
|
98007
|
-
return;
|
|
98008
|
-
return ownedPath;
|
|
97830
|
+
|
|
97831
|
+
// src/indexer/units/unit.ts
|
|
97832
|
+
init_metadata();
|
|
97833
|
+
var UNIT_HEADER_SECTION_SEPARATOR = " › ";
|
|
97834
|
+
function structuredFieldsText(source) {
|
|
97835
|
+
const body = [source.description, source.tags, source.hints, source.parameters].filter((field) => field.length > 0).join(`
|
|
97836
|
+
`);
|
|
97837
|
+
return `${source.name}
|
|
97838
|
+
${body}`;
|
|
98009
97839
|
}
|
|
98010
|
-
function
|
|
98011
|
-
if (
|
|
98012
|
-
return
|
|
98013
|
-
|
|
98014
|
-
|
|
98015
|
-
|
|
98016
|
-
|
|
98017
|
-
}
|
|
98018
|
-
|
|
98019
|
-
|
|
98020
|
-
const
|
|
98021
|
-
|
|
98022
|
-
|
|
98023
|
-
|
|
98024
|
-
|
|
98025
|
-
|
|
98026
|
-
|
|
98027
|
-
|
|
98028
|
-
|
|
98029
|
-
|
|
98030
|
-
const canonicalName = canonicalizeWorkflowName(authoredName);
|
|
98031
|
-
if (!isSafeRelativeName(canonicalName))
|
|
98032
|
-
continue;
|
|
98033
|
-
const extension = path60.extname(normalizedSourcePath);
|
|
98034
|
-
const lowerExtension = extension.toLowerCase();
|
|
98035
|
-
if (!WORKFLOW_EXTENSIONS.includes(lowerExtension))
|
|
98036
|
-
continue;
|
|
98037
|
-
const candidate = {
|
|
98038
|
-
path: normalizedSourcePath,
|
|
98039
|
-
relativePath: toPosix(path60.relative(authoredRoot, normalizedSourcePath)),
|
|
98040
|
-
lowerExtension,
|
|
98041
|
-
extensionlessStem: path60.basename(normalizedSourcePath).slice(0, -extension.length)
|
|
98042
|
-
};
|
|
98043
|
-
const domain = candidatesByName.get(canonicalName) ?? [];
|
|
98044
|
-
domain.push(candidate);
|
|
98045
|
-
candidatesByName.set(canonicalName, domain);
|
|
98046
|
-
}
|
|
98047
|
-
const resolutions = [];
|
|
98048
|
-
for (const canonicalName of [...candidatesByName.keys()].sort(compareCodePoints)) {
|
|
98049
|
-
const candidates = candidatesByName.get(canonicalName) ?? [];
|
|
98050
|
-
candidates.sort((left, right) => compareCodePoints(left.relativePath, right.relativePath));
|
|
98051
|
-
const sources = inspectWorkflowSourceDomain(candidates, canonicalName, realRoot);
|
|
98052
|
-
const sourcePaths2 = candidates.map((candidate) => candidate.relativePath);
|
|
98053
|
-
resolutions.push({
|
|
98054
|
-
canonicalName,
|
|
98055
|
-
sourcePaths: sourcePaths2,
|
|
98056
|
-
source: pickWorkflowSource(adapterId, canonicalName, sources)
|
|
98057
|
-
});
|
|
98058
|
-
}
|
|
98059
|
-
return resolutions;
|
|
97840
|
+
function parametersText(parameters) {
|
|
97841
|
+
if (!parameters || parameters.length === 0)
|
|
97842
|
+
return "";
|
|
97843
|
+
return parameters.map((param) => param.description ? `${param.name}: ${param.description}` : param.name).join(`
|
|
97844
|
+
`).toLowerCase();
|
|
97845
|
+
}
|
|
97846
|
+
function fragmentHeaderText(name, sectionTitle) {
|
|
97847
|
+
return sectionTitle ? `${name}${UNIT_HEADER_SECTION_SEPARATOR}${sectionTitle}` : name;
|
|
97848
|
+
}
|
|
97849
|
+
function fragmentSectionTitles(safeMarkdown, fragments) {
|
|
97850
|
+
const headings = parseMarkdownToc(safeMarkdown).headings;
|
|
97851
|
+
let headingIndex = 0;
|
|
97852
|
+
let current = null;
|
|
97853
|
+
return fragments.map((fragment) => {
|
|
97854
|
+
while (headingIndex < headings.length && headings[headingIndex].line <= fragment.startLine) {
|
|
97855
|
+
current = headings[headingIndex].text;
|
|
97856
|
+
headingIndex++;
|
|
97857
|
+
}
|
|
97858
|
+
return current;
|
|
97859
|
+
});
|
|
98060
97860
|
}
|
|
98061
|
-
function
|
|
98062
|
-
const
|
|
98063
|
-
|
|
98064
|
-
|
|
98065
|
-
|
|
98066
|
-
|
|
97861
|
+
function splitOverflowingText(text, maxChars) {
|
|
97862
|
+
const pieces = [];
|
|
97863
|
+
let rest = text;
|
|
97864
|
+
while (rest.length > maxChars) {
|
|
97865
|
+
const window2 = rest.slice(0, maxChars);
|
|
97866
|
+
const newlineCut = window2.lastIndexOf(`
|
|
97867
|
+
`);
|
|
97868
|
+
if (newlineCut > 0) {
|
|
97869
|
+
pieces.push(rest.slice(0, newlineCut));
|
|
97870
|
+
rest = rest.slice(newlineCut + 1);
|
|
98067
97871
|
continue;
|
|
98068
97872
|
}
|
|
98069
|
-
|
|
98070
|
-
|
|
97873
|
+
const spaceCut = window2.lastIndexOf(" ");
|
|
97874
|
+
if (spaceCut > 0) {
|
|
97875
|
+
pieces.push(rest.slice(0, spaceCut));
|
|
97876
|
+
rest = rest.slice(spaceCut + 1);
|
|
97877
|
+
continue;
|
|
98071
97878
|
}
|
|
97879
|
+
pieces.push(rest.slice(0, maxChars));
|
|
97880
|
+
rest = rest.slice(maxChars);
|
|
97881
|
+
}
|
|
97882
|
+
if (rest.length > 0)
|
|
97883
|
+
pieces.push(rest);
|
|
97884
|
+
return pieces;
|
|
97885
|
+
}
|
|
97886
|
+
function deriveUnits(source, maxChars) {
|
|
97887
|
+
if (!Number.isFinite(maxChars) || maxChars <= 0) {
|
|
97888
|
+
throw new RangeError("deriveUnits: maxChars must be a positive finite number");
|
|
97889
|
+
}
|
|
97890
|
+
const units = [];
|
|
97891
|
+
let ordinal = 0;
|
|
97892
|
+
const pushUnit = (fragmentId, text) => {
|
|
97893
|
+
units.push({ entryId: source.entryId, ordinal: ordinal++, fragmentId, hash: hashEmbeddableText(text), text });
|
|
97894
|
+
};
|
|
97895
|
+
for (const text of splitOverflowingText(structuredFieldsText(source), maxChars))
|
|
97896
|
+
pushUnit(null, text);
|
|
97897
|
+
if (source.safeMarkdown != null) {
|
|
97898
|
+
const fragments = splitMarkdownFragments(source.safeMarkdown);
|
|
97899
|
+
const sectionTitles = fragmentSectionTitles(source.safeMarkdown, fragments);
|
|
97900
|
+
fragments.forEach((fragment, index) => {
|
|
97901
|
+
const header = fragmentHeaderText(source.name, sectionTitles[index] ?? null);
|
|
97902
|
+
const text = `${header}
|
|
97903
|
+
${fragment.text}`;
|
|
97904
|
+
for (const piece of splitOverflowingText(text, maxChars))
|
|
97905
|
+
pushUnit(fragment.fragmentId, piece);
|
|
97906
|
+
});
|
|
98072
97907
|
}
|
|
98073
|
-
return
|
|
98074
|
-
}
|
|
98075
|
-
function pickWorkflowSource(adapterId, canonicalName, sources) {
|
|
98076
|
-
if (sources.length <= 1)
|
|
98077
|
-
return sources[0];
|
|
98078
|
-
const winner = [...sources].sort((left, right) => left.format === right.format ? 0 : left.format === "markdown" ? -1 : 1)[0];
|
|
98079
|
-
const shadowed = sources.filter((source) => source !== winner);
|
|
98080
|
-
const displayName = adapterId === "akm" ? `workflows/${canonicalName}` : canonicalName;
|
|
98081
|
-
warnOnce(`workflow-source-collision:${displayName}`, `Workflow "${displayName}" has both a .md and .yml source (${shadowed.map((source) => source.relativePath).join(", ")} shadowed by ${winner?.relativePath}); using the .md source. Remove the shadowed sibling to ` + "silence this warning.");
|
|
98082
|
-
return winner;
|
|
97908
|
+
return units;
|
|
98083
97909
|
}
|
|
98084
|
-
function
|
|
98085
|
-
const
|
|
98086
|
-
|
|
98087
|
-
try {
|
|
98088
|
-
authoredStat = fs49.lstatSync(candidate.path);
|
|
98089
|
-
} catch {
|
|
98090
|
-
issues.push(new WorkflowSourceLinkResolutionError(candidate.relativePath));
|
|
98091
|
-
return { issues };
|
|
98092
|
-
}
|
|
98093
|
-
const isLink = authoredStat.isSymbolicLink();
|
|
98094
|
-
if (!isLink && !authoredStat.isFile()) {
|
|
98095
|
-
issues.push(new WorkflowSourceLinkResolutionError(candidate.relativePath));
|
|
98096
|
-
return { issues };
|
|
98097
|
-
}
|
|
98098
|
-
let realPath;
|
|
98099
|
-
try {
|
|
98100
|
-
realPath = fs49.realpathSync(candidate.path);
|
|
98101
|
-
} catch {
|
|
98102
|
-
issues.push(new WorkflowSourceLinkResolutionError(candidate.relativePath));
|
|
98103
|
-
return { issues };
|
|
98104
|
-
}
|
|
98105
|
-
const targetPath = toPosix(path60.relative(realRoot, realPath));
|
|
98106
|
-
const contained2 = isWithinResolved(realPath, realRoot);
|
|
98107
|
-
if (!contained2)
|
|
98108
|
-
issues.push(new WorkflowSourcePathIdentityError(candidate.relativePath, targetPath));
|
|
98109
|
-
if (isLink && path60.extname(realPath).toLowerCase() !== candidate.lowerExtension) {
|
|
98110
|
-
issues.push(new WorkflowSourceLinkIdentityError(candidate.relativePath, targetPath));
|
|
98111
|
-
}
|
|
98112
|
-
if (contained2) {
|
|
98113
|
-
try {
|
|
98114
|
-
if (!fs49.statSync(realPath).isFile())
|
|
98115
|
-
issues.push(new WorkflowSourceLinkResolutionError(candidate.relativePath));
|
|
98116
|
-
} catch {
|
|
98117
|
-
issues.push(new WorkflowSourceLinkResolutionError(candidate.relativePath));
|
|
98118
|
-
}
|
|
98119
|
-
}
|
|
98120
|
-
if (issues.length > 0)
|
|
98121
|
-
return { issues };
|
|
97910
|
+
function toUnitSource(entryId, entry) {
|
|
97911
|
+
const fields = buildSearchFields(entry);
|
|
97912
|
+
const safeMarkdown = hasMarkdownFragmentContent(entry) ? getMarkdownFragmentContent(entry) ?? null : typeof entry.content === "string" && entry.content.trim() ? entry.content : null;
|
|
98122
97913
|
return {
|
|
98123
|
-
|
|
98124
|
-
|
|
98125
|
-
|
|
98126
|
-
|
|
98127
|
-
|
|
98128
|
-
|
|
98129
|
-
|
|
98130
|
-
}
|
|
97914
|
+
entryId,
|
|
97915
|
+
name: fields.name,
|
|
97916
|
+
description: fields.description,
|
|
97917
|
+
tags: fields.tags,
|
|
97918
|
+
hints: fields.hints,
|
|
97919
|
+
parameters: parametersText(entry.parameters),
|
|
97920
|
+
safeMarkdown
|
|
98131
97921
|
};
|
|
98132
97922
|
}
|
|
98133
|
-
function isSafeRelativeName(name) {
|
|
98134
|
-
return name.length > 0 && !path60.posix.isAbsolute(name) && name !== ".." && !name.startsWith("../") && path60.posix.normalize(name) === name;
|
|
98135
|
-
}
|
|
98136
|
-
function isWithinResolved(candidate, root) {
|
|
98137
|
-
const relative = path60.relative(root, path60.resolve(candidate));
|
|
98138
|
-
return relative === "" || !relative.startsWith("..") && !path60.isAbsolute(relative);
|
|
98139
|
-
}
|
|
98140
97923
|
|
|
98141
|
-
// src/indexer/
|
|
97924
|
+
// src/indexer/reconcile.ts
|
|
97925
|
+
init_metadata();
|
|
97926
|
+
|
|
97927
|
+
// src/indexer/scan/parse-file.ts
|
|
98142
97928
|
init_compile();
|
|
98143
97929
|
init_metadata();
|
|
98144
|
-
init_file_context();
|
|
98145
97930
|
|
|
98146
97931
|
// src/indexer/scan/doc-to-entry.ts
|
|
98147
97932
|
init_metadata();
|
|
98148
|
-
import
|
|
97933
|
+
import path60 from "node:path";
|
|
98149
97934
|
function indexDocumentToStashEntry(doc) {
|
|
98150
97935
|
const dj = doc.documentJson ?? {};
|
|
98151
97936
|
const entry = {
|
|
98152
97937
|
name: doc.name,
|
|
98153
97938
|
type: doc.type,
|
|
98154
|
-
filename:
|
|
97939
|
+
filename: path60.basename(doc.path ?? "")
|
|
98155
97940
|
};
|
|
98156
97941
|
if (doc.description !== undefined)
|
|
98157
97942
|
entry.description = doc.description;
|
|
@@ -98247,58 +98032,25 @@ function isIntent(value) {
|
|
|
98247
98032
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
98248
98033
|
}
|
|
98249
98034
|
|
|
98250
|
-
// src/indexer/scan/
|
|
98035
|
+
// src/indexer/scan/parse-file.ts
|
|
98251
98036
|
var WORKFLOW_MD_RENDERER = "workflow-md";
|
|
98252
|
-
function
|
|
98253
|
-
const
|
|
98254
|
-
|
|
98255
|
-
|
|
98256
|
-
|
|
98257
|
-
|
|
98258
|
-
|
|
98259
|
-
|
|
98260
|
-
|
|
98261
|
-
|
|
98262
|
-
const leftName = workflowNameForSourcePath(component.root, adapter.id, left.absPath);
|
|
98263
|
-
const rightName = workflowNameForSourcePath(component.root, adapter.id, right.absPath);
|
|
98264
|
-
const leftOwner = leftName !== undefined && workflowOwnerPathByCanonicalName.get(canonicalizeWorkflowName(leftName)) === path63.resolve(left.absPath);
|
|
98265
|
-
const rightOwner = rightName !== undefined && workflowOwnerPathByCanonicalName.get(canonicalizeWorkflowName(rightName)) === path63.resolve(right.absPath);
|
|
98266
|
-
if (leftOwner !== rightOwner)
|
|
98267
|
-
return leftOwner ? -1 : 1;
|
|
98268
|
-
return compareCodePoints(left.absPath, right.absPath);
|
|
98269
|
-
});
|
|
98270
|
-
for (const file of orderedFileContexts) {
|
|
98271
|
-
if (rejectedPaths.has(file.absPath))
|
|
98272
|
-
continue;
|
|
98273
|
-
const workflowName = workflowNameForSourcePath(component.root, adapter.id, file.absPath);
|
|
98274
|
-
if (workflowName !== undefined) {
|
|
98275
|
-
const canonicalName = canonicalizeWorkflowName(workflowName);
|
|
98276
|
-
const ownerPath = workflowOwnerPathByCanonicalName.get(canonicalName);
|
|
98277
|
-
if (ownerPath !== undefined && ownerPath !== path63.resolve(file.absPath) && !invalidWorkflowOwnerNames.has(canonicalName)) {
|
|
98278
|
-
continue;
|
|
98279
|
-
}
|
|
98280
|
-
}
|
|
98281
|
-
const doc = adapter.recognize(component, file);
|
|
98282
|
-
if (doc === null)
|
|
98283
|
-
continue;
|
|
98284
|
-
if (!doc.conceptId) {
|
|
98285
|
-
warnings.push(`Skipped ${file.absPath}: adapter "${adapter.id}" returned no conceptId.`);
|
|
98286
|
-
continue;
|
|
98287
|
-
}
|
|
98288
|
-
const entry = indexDocumentToStashEntry(doc);
|
|
98289
|
-
const dropWarning = handleWorkflowDoc(doc, file, component.root);
|
|
98290
|
-
if (dropWarning !== null) {
|
|
98291
|
-
warnings.push(dropWarning);
|
|
98292
|
-
if (workflowName !== undefined)
|
|
98293
|
-
invalidWorkflowOwnerNames.add(canonicalizeWorkflowName(workflowName));
|
|
98294
|
-
continue;
|
|
98295
|
-
}
|
|
98296
|
-
if (doc.hash !== undefined)
|
|
98297
|
-
hashByFile.set(file.absPath, doc.hash);
|
|
98298
|
-
conceptIdByFile.set(file.absPath, doc.conceptId);
|
|
98299
|
-
entries.push(entry);
|
|
98037
|
+
function parseFileDocument(adapter, component, file) {
|
|
98038
|
+
const doc = adapter.recognize(component, file);
|
|
98039
|
+
if (doc === null)
|
|
98040
|
+
return { parsed: null, warning: null, isWorkflowDrop: false };
|
|
98041
|
+
if (!doc.conceptId) {
|
|
98042
|
+
return {
|
|
98043
|
+
parsed: null,
|
|
98044
|
+
warning: `Skipped ${file.absPath}: adapter "${adapter.id}" returned no conceptId.`,
|
|
98045
|
+
isWorkflowDrop: false
|
|
98046
|
+
};
|
|
98300
98047
|
}
|
|
98301
|
-
|
|
98048
|
+
const entry = indexDocumentToStashEntry(doc);
|
|
98049
|
+
const dropWarning = handleWorkflowDoc(doc, file, component.root);
|
|
98050
|
+
if (dropWarning !== null) {
|
|
98051
|
+
return { parsed: null, warning: dropWarning, isWorkflowDrop: true };
|
|
98052
|
+
}
|
|
98053
|
+
return { parsed: { entry, hash: doc.hash, conceptId: doc.conceptId }, warning: null, isWorkflowDrop: false };
|
|
98302
98054
|
}
|
|
98303
98055
|
function handleWorkflowDoc(doc, file, workspaceRoot) {
|
|
98304
98056
|
if (doc.type !== "workflow" || doc.adapterId !== "akm" && doc.adapterId !== "akm-workflow" || docRenderer(doc) !== WORKFLOW_MD_RENDERER && doc.adapterId !== "akm-workflow") {
|
|
@@ -98324,114 +98076,265 @@ function workflowDropWarning(file, errors3) {
|
|
|
98324
98076
|
${summary}`);
|
|
98325
98077
|
}
|
|
98326
98078
|
|
|
98327
|
-
// src/indexer/
|
|
98079
|
+
// src/indexer/reconcile.ts
|
|
98328
98080
|
init_file_context();
|
|
98081
|
+
|
|
98082
|
+
// src/indexer/walk/walker.ts
|
|
98083
|
+
init_asset_placement();
|
|
98084
|
+
init_recognition_util();
|
|
98085
|
+
init_file_context();
|
|
98086
|
+
var ALWAYS_SKIP_DIRS = new Set([".git"]);
|
|
98087
|
+
var AKM_SKIP_DIRS = new Set(["node_modules", "bin", ".cache"]);
|
|
98088
|
+
|
|
98089
|
+
// src/indexer/reconcile.ts
|
|
98090
|
+
async function reconcilePaths(db, paths, bundleId, opts) {
|
|
98091
|
+
const counts = emptyCounts();
|
|
98092
|
+
if (paths.length === 0)
|
|
98093
|
+
return counts;
|
|
98094
|
+
const config = loadConfig();
|
|
98095
|
+
let rootPath;
|
|
98096
|
+
let ctx;
|
|
98097
|
+
if (opts?.root) {
|
|
98098
|
+
const resolvedCtx = resolveRootContext(opts.root, bundleId);
|
|
98099
|
+
if (!resolvedCtx)
|
|
98100
|
+
return counts;
|
|
98101
|
+
rootPath = opts.root;
|
|
98102
|
+
ctx = resolvedCtx;
|
|
98103
|
+
} else {
|
|
98104
|
+
const resolvedRoot = resolveBundleRoot(bundleId, config);
|
|
98105
|
+
if (!resolvedRoot)
|
|
98106
|
+
return counts;
|
|
98107
|
+
rootPath = resolvedRoot.rootPath;
|
|
98108
|
+
ctx = { bundleId, component: resolvedRoot.component, adapter: resolvedRoot.adapter };
|
|
98109
|
+
}
|
|
98110
|
+
const maxChars = unitMaxChars(await probeProviderLimits(config.embedding ?? {}));
|
|
98111
|
+
for (const rawPath of paths) {
|
|
98112
|
+
counts.scanned++;
|
|
98113
|
+
const absPath = path62.resolve(rawPath);
|
|
98114
|
+
if (!fs51.existsSync(absPath)) {
|
|
98115
|
+
if (deleteFileAndEntryByPath(db, absPath))
|
|
98116
|
+
counts.removed++;
|
|
98117
|
+
continue;
|
|
98118
|
+
}
|
|
98119
|
+
const file = buildFileContext(rootPath, absPath);
|
|
98120
|
+
const classified = classifyFile(ctx, file, getFileState(db, absPath), (message) => counts.warnings.push(message));
|
|
98121
|
+
if (classified === "unchanged")
|
|
98122
|
+
counts.unchanged++;
|
|
98123
|
+
else if (classified === "unindexable") {
|
|
98124
|
+
if (deleteFileAndEntryByPath(db, absPath))
|
|
98125
|
+
counts.removed++;
|
|
98126
|
+
} else {
|
|
98127
|
+
applyOutcome(counts, applyChange(db, ctx, classified, maxChars, undefined, true));
|
|
98128
|
+
}
|
|
98129
|
+
}
|
|
98130
|
+
return counts;
|
|
98131
|
+
}
|
|
98132
|
+
function resolveRootContext(rootPath, bundleId) {
|
|
98133
|
+
const component = deriveInstallations([
|
|
98134
|
+
{ path: rootPath, registryId: bundleId, writable: true, adapterId: configuredAdapterIdForBundle(bundleId) }
|
|
98135
|
+
])[0]?.components[0];
|
|
98136
|
+
if (!component)
|
|
98137
|
+
return;
|
|
98138
|
+
const adapter = adapterForId(component.adapter);
|
|
98139
|
+
if (!adapter)
|
|
98140
|
+
return;
|
|
98141
|
+
return { bundleId, component, adapter };
|
|
98142
|
+
}
|
|
98143
|
+
function configuredAdapterIdForBundle(bundleId) {
|
|
98144
|
+
const bundle = loadConfig().bundles?.[bundleId];
|
|
98145
|
+
if (!bundle)
|
|
98146
|
+
return;
|
|
98147
|
+
return Object.values(bundle.components ?? {})[0]?.adapter;
|
|
98148
|
+
}
|
|
98149
|
+
function resolveBundleRoot(bundleId, config) {
|
|
98150
|
+
const sources = resolveSourceEntries(undefined, config);
|
|
98151
|
+
const installations = deriveInstallations(sources);
|
|
98152
|
+
const index = installations.findIndex((installation) => installation.id === bundleId);
|
|
98153
|
+
const source = index === -1 ? undefined : sources[index];
|
|
98154
|
+
const component = index === -1 ? undefined : installations[index]?.components[0];
|
|
98155
|
+
if (!source || !component)
|
|
98156
|
+
return;
|
|
98157
|
+
const adapter = adapterForId(component.adapter);
|
|
98158
|
+
if (!adapter)
|
|
98159
|
+
return;
|
|
98160
|
+
return { rootPath: source.path, component, adapter };
|
|
98161
|
+
}
|
|
98162
|
+
function classifyFile(ctx, file, storedHint, onWarning) {
|
|
98163
|
+
let stat;
|
|
98164
|
+
try {
|
|
98165
|
+
stat = file.stat();
|
|
98166
|
+
} catch {
|
|
98167
|
+
const access = classifyPathAccess(file.absPath);
|
|
98168
|
+
if (access.access === "inaccessible") {
|
|
98169
|
+
if (!storedHint) {
|
|
98170
|
+
onWarning?.(`New file akm cannot read, never indexed: ${describeInaccessiblePath(file.absPath, access.code)}`, false);
|
|
98171
|
+
}
|
|
98172
|
+
return "unchanged";
|
|
98173
|
+
}
|
|
98174
|
+
return "unindexable";
|
|
98175
|
+
}
|
|
98176
|
+
if (storedHint && storedHint.size === stat.size && storedHint.mtimeMs === stat.mtimeMs && storedHint.ctimeMs === stat.ctimeMs && storedHint.adapterId === ctx.adapter.id) {
|
|
98177
|
+
return "unchanged";
|
|
98178
|
+
}
|
|
98179
|
+
const outcome = parseFileDocument(ctx.adapter, ctx.component, file);
|
|
98180
|
+
if (outcome.parsed === null) {
|
|
98181
|
+
if (outcome.warning !== null)
|
|
98182
|
+
onWarning?.(outcome.warning, outcome.isWorkflowDrop);
|
|
98183
|
+
return "unindexable";
|
|
98184
|
+
}
|
|
98185
|
+
const hash4 = outcome.parsed.hash ?? hashContent(file.content());
|
|
98186
|
+
return { file, stat, entry: outcome.parsed.entry, conceptId: outcome.parsed.conceptId, hash: hash4 };
|
|
98187
|
+
}
|
|
98188
|
+
function applyChange(db, ctx, change, maxChars, renameSource, supersedeOtherBundles) {
|
|
98189
|
+
const { file, stat, entry, conceptId, hash: hash4 } = change;
|
|
98190
|
+
const searchText = buildSearchText(entry);
|
|
98191
|
+
const provenance = deriveEntryProvenance({ bundleId: ctx.bundleId, componentId: ctx.component.id, adapterId: ctx.component.adapter }, entry.type, entry.name, conceptId);
|
|
98192
|
+
const entryWithSize = { ...entry, fileSize: stat.size };
|
|
98193
|
+
if (hasMarkdownFragmentContent(entry))
|
|
98194
|
+
setMarkdownFragmentContent(entryWithSize, getMarkdownFragmentContent(entry));
|
|
98195
|
+
let orphanedUnitHashes = [];
|
|
98196
|
+
const result = withImmediateTransaction(db, () => {
|
|
98197
|
+
if (supersedeOtherBundles)
|
|
98198
|
+
supersedeOtherItemRefsAtPath(db, file.absPath, provenance.itemRef);
|
|
98199
|
+
const written = renameSource ? repointOrInsert(db, renameSource.path, file.absPath, entryWithSize, searchText, provenance, hash4) : upsertOrInsert(db, file.absPath, entryWithSize, searchText, provenance, hash4);
|
|
98200
|
+
if (renameSource)
|
|
98201
|
+
deleteFileStates(db, [renameSource.path]);
|
|
98202
|
+
upsertFileState(db, {
|
|
98203
|
+
path: file.absPath,
|
|
98204
|
+
bundleId: ctx.bundleId,
|
|
98205
|
+
size: stat.size,
|
|
98206
|
+
mtimeMs: stat.mtimeMs,
|
|
98207
|
+
ctimeMs: stat.ctimeMs,
|
|
98208
|
+
blobHash: hash4,
|
|
98209
|
+
adapterId: ctx.adapter.id
|
|
98210
|
+
});
|
|
98211
|
+
const units = deriveUnits(toUnitSource(written.entryId, entry), maxChars);
|
|
98212
|
+
const { inserted } = insertNewUnitTexts(db, units.map((unit) => ({
|
|
98213
|
+
hash: unit.hash,
|
|
98214
|
+
kind: unit.fragmentId === null ? "card" : "fragment",
|
|
98215
|
+
text: unit.text
|
|
98216
|
+
})));
|
|
98217
|
+
const previousHashes = db.prepare("SELECT unit_hash FROM entry_units WHERE entry_id = ?").all(written.entryId).map((row) => row.unit_hash);
|
|
98218
|
+
replaceEntryUnits(db, written.entryId, units.map((unit) => ({ ordinal: unit.ordinal, fragmentId: unit.fragmentId, hash: unit.hash })));
|
|
98219
|
+
const currentHashes = new Set(units.map((unit) => unit.hash));
|
|
98220
|
+
orphanedUnitHashes = previousHashes.filter((oldHash) => !currentHashes.has(oldHash));
|
|
98221
|
+
return {
|
|
98222
|
+
outcome: written.outcome,
|
|
98223
|
+
unitsAdded: inserted,
|
|
98224
|
+
entryId: written.entryId,
|
|
98225
|
+
entry: entryWithSize,
|
|
98226
|
+
provenance
|
|
98227
|
+
};
|
|
98228
|
+
}, "index");
|
|
98229
|
+
if (orphanedUnitHashes.length > 0)
|
|
98230
|
+
pruneOrphanUnitTextsForHashes(db, orphanedUnitHashes);
|
|
98231
|
+
return result;
|
|
98232
|
+
}
|
|
98233
|
+
function upsertOrInsert(db, filePath, entry, searchText, provenance, hash4) {
|
|
98234
|
+
const existedBefore = getFileState(db, filePath) !== undefined;
|
|
98235
|
+
const entryId = upsertEntry(db, filePath, entry, searchText, provenance, hash4);
|
|
98236
|
+
return { entryId, outcome: existedBefore ? "changed" : "added" };
|
|
98237
|
+
}
|
|
98238
|
+
function repointOrInsert(db, oldPath, newPath, entry, searchText, provenance, hash4) {
|
|
98239
|
+
const oldRow = db.prepare("SELECT id FROM entries WHERE file_path = ?").get(oldPath);
|
|
98240
|
+
if (!oldRow)
|
|
98241
|
+
return upsertOrInsert(db, newPath, entry, searchText, provenance, hash4);
|
|
98242
|
+
const claimedByOther = db.prepare("SELECT 1 FROM entries WHERE item_ref = ? AND id <> ?").get(provenance.itemRef, oldRow.id);
|
|
98243
|
+
if (claimedByOther)
|
|
98244
|
+
return upsertOrInsert(db, newPath, entry, searchText, provenance, hash4);
|
|
98245
|
+
repointEntry(db, oldRow.id, newPath, entry, searchText, provenance, hash4);
|
|
98246
|
+
return { entryId: oldRow.id, outcome: "changed" };
|
|
98247
|
+
}
|
|
98248
|
+
function repointEntry(db, entryId, filePath, entry, searchText, provenance, contentHash) {
|
|
98249
|
+
const derivedFrom = typeof entry.derivedFrom === "string" && entry.derivedFrom.trim() ? entry.derivedFrom.trim() : null;
|
|
98250
|
+
db.prepare(`UPDATE entries SET item_ref = ?, bundle_id = ?, component_id = ?, concept_id = ?, adapter_id = ?, type = ?,
|
|
98251
|
+
file_path = ?, content_hash = ?, document_json = ?, search_text = ?, derived_from = ?
|
|
98252
|
+
WHERE id = ?`).run(provenance.itemRef, provenance.bundleId, provenance.componentId, provenance.conceptId, provenance.adapterId, entry.type, filePath, contentHash, JSON.stringify(entry), searchText, derivedFrom, entryId);
|
|
98253
|
+
replaceFragmentSource(db, entryId, hasMarkdownFragmentContent(entry) ? getMarkdownFragmentContent(entry) ?? null : undefined);
|
|
98254
|
+
}
|
|
98255
|
+
function supersedeOtherItemRefsAtPath(db, filePath, keepItemRef) {
|
|
98256
|
+
const staleIds = db.prepare("SELECT id FROM entries WHERE file_path = ? AND item_ref <> ?").all(filePath, keepItemRef).map((row) => row.id);
|
|
98257
|
+
if (staleIds.length > 0)
|
|
98258
|
+
deleteEntriesByIds(db, staleIds);
|
|
98259
|
+
}
|
|
98260
|
+
function deleteFileAndEntryByPath(db, filePath) {
|
|
98261
|
+
return withImmediateTransaction(db, () => {
|
|
98262
|
+
const entryIds = db.prepare("SELECT id FROM entries WHERE file_path = ?").all(filePath).map((row) => row.id);
|
|
98263
|
+
if (entryIds.length > 0)
|
|
98264
|
+
deleteEntriesByIds(db, entryIds);
|
|
98265
|
+
const hadFileRow = getFileState(db, filePath) !== undefined;
|
|
98266
|
+
if (hadFileRow)
|
|
98267
|
+
deleteFileStates(db, [filePath]);
|
|
98268
|
+
return entryIds.length > 0 || hadFileRow;
|
|
98269
|
+
}, "index");
|
|
98270
|
+
}
|
|
98271
|
+
function emptyCounts() {
|
|
98272
|
+
return { scanned: 0, unchanged: 0, added: 0, changed: 0, removed: 0, unitsAdded: 0, complete: true, warnings: [] };
|
|
98273
|
+
}
|
|
98274
|
+
function applyOutcome(counts, result) {
|
|
98275
|
+
if (result.outcome === "added")
|
|
98276
|
+
counts.added++;
|
|
98277
|
+
else
|
|
98278
|
+
counts.changed++;
|
|
98279
|
+
counts.unitsAdded += result.unitsAdded;
|
|
98280
|
+
}
|
|
98281
|
+
|
|
98282
|
+
// src/indexer/index-written-assets.ts
|
|
98329
98283
|
var WRITE_PATH_INDEX_BUSY_TIMEOUT_MS = 5000;
|
|
98284
|
+
function unitHashesForFiles(db, files) {
|
|
98285
|
+
const hashes = new Set;
|
|
98286
|
+
for (let offset = 0;offset < files.length; offset += SQLITE_CHUNK_SIZE) {
|
|
98287
|
+
const chunk2 = files.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
98288
|
+
const placeholders = chunk2.map(() => "?").join(",");
|
|
98289
|
+
const rows = db.prepare(`SELECT DISTINCT eu.unit_hash AS unitHash
|
|
98290
|
+
FROM entries e
|
|
98291
|
+
JOIN entry_units eu ON eu.entry_id = e.id
|
|
98292
|
+
WHERE e.file_path IN (${placeholders})`).all(...chunk2);
|
|
98293
|
+
for (const row of rows)
|
|
98294
|
+
hashes.add(row.unitHash);
|
|
98295
|
+
}
|
|
98296
|
+
return [...hashes];
|
|
98297
|
+
}
|
|
98298
|
+
function resolveWrittenAssetBundleId(stashDir) {
|
|
98299
|
+
const resolvedStashDir = path63.resolve(stashDir);
|
|
98300
|
+
const sourceEntries = resolveSourceEntries();
|
|
98301
|
+
const index = sourceEntries.findIndex((entry) => path63.resolve(entry.path) === resolvedStashDir);
|
|
98302
|
+
if (index !== -1)
|
|
98303
|
+
return deriveInstallations(sourceEntries)[index]?.id;
|
|
98304
|
+
return deriveInstallations([{ path: stashDir, writable: true }])[0]?.id;
|
|
98305
|
+
}
|
|
98330
98306
|
async function indexWrittenAssets(stashDir, filePaths, options = {}) {
|
|
98331
98307
|
try {
|
|
98332
98308
|
return await (async () => {
|
|
98333
|
-
const rebuildProbe = probeLock(getIndexRebuildLockPath());
|
|
98334
|
-
if (rebuildProbe.state === "held") {
|
|
98335
|
-
const holderLabel = formatLockHolderPid({
|
|
98336
|
-
pid: rebuildProbe.holderPid,
|
|
98337
|
-
launcherPid: rebuildProbe.launcherPid ?? null
|
|
98338
|
-
});
|
|
98339
|
-
warn(`index rebuild in progress (pid ${holderLabel}); the next index pass will index ${filePaths.join(", ")}`);
|
|
98340
|
-
return true;
|
|
98341
|
-
}
|
|
98342
98309
|
const dbPath = getDbPath();
|
|
98343
98310
|
if (isPathAbsent(dbPath))
|
|
98344
98311
|
return true;
|
|
98345
98312
|
const files = filePaths.filter((f) => {
|
|
98346
|
-
const rel =
|
|
98313
|
+
const rel = path63.relative(stashDir, f);
|
|
98347
98314
|
return !rel.split(/[\\/]+/).some((segment) => segment.startsWith("."));
|
|
98348
98315
|
});
|
|
98349
98316
|
if (files.length === 0)
|
|
98350
98317
|
return true;
|
|
98351
|
-
const
|
|
98352
|
-
|
|
98353
|
-
])[0]?.components[0];
|
|
98354
|
-
if (!component)
|
|
98318
|
+
const bundleId = options.bundleId ?? resolveWrittenAssetBundleId(stashDir);
|
|
98319
|
+
if (!bundleId)
|
|
98355
98320
|
throw new Error(`Could not derive bundle provenance for ${stashDir}`);
|
|
98356
|
-
const pairs = [];
|
|
98357
|
-
const unindexable = new Set;
|
|
98358
|
-
const rejectedConceptIds = new Set;
|
|
98359
|
-
for (const file of files) {
|
|
98360
|
-
if (!fs53.existsSync(file)) {
|
|
98361
|
-
let authoredDanglingSymlink = false;
|
|
98362
|
-
try {
|
|
98363
|
-
authoredDanglingSymlink = fs53.lstatSync(file).isSymbolicLink();
|
|
98364
|
-
} catch {}
|
|
98365
|
-
if (!authoredDanglingSymlink) {
|
|
98366
|
-
unindexable.add(file);
|
|
98367
|
-
continue;
|
|
98368
|
-
}
|
|
98369
|
-
}
|
|
98370
|
-
const ctx = buildFileContext(stashDir, file);
|
|
98371
|
-
const drained = drainDirDocuments(akmAdapter, component, [ctx]);
|
|
98372
|
-
for (const rejectedPath of drained.rejectedPaths)
|
|
98373
|
-
unindexable.add(rejectedPath);
|
|
98374
|
-
for (const conceptId2 of drained.rejectedConceptIds)
|
|
98375
|
-
rejectedConceptIds.add(conceptId2);
|
|
98376
|
-
const entry = drained.entries[0];
|
|
98377
|
-
const conceptId = drained.conceptIdByFile.get(ctx.absPath);
|
|
98378
|
-
if (entry && conceptId)
|
|
98379
|
-
pairs.push({ file, entry, conceptId, contentHash: drained.hashByFile.get(ctx.absPath) });
|
|
98380
|
-
else
|
|
98381
|
-
unindexable.add(file);
|
|
98382
|
-
}
|
|
98383
98321
|
const db = openExistingDatabase(dbPath);
|
|
98384
98322
|
try {
|
|
98385
98323
|
db.exec(`PRAGMA busy_timeout = ${WRITE_PATH_INDEX_BUSY_TIMEOUT_MS}`);
|
|
98386
98324
|
if (getEntryCount(db) === 0)
|
|
98387
98325
|
return true;
|
|
98388
|
-
|
|
98389
|
-
|
|
98390
|
-
db.transaction(() => {
|
|
98391
|
-
const unindexableEntryIds = new Set;
|
|
98392
|
-
for (const file of unindexable) {
|
|
98393
|
-
const rows = db.prepare(`SELECT id FROM entries
|
|
98394
|
-
WHERE file_path = ?
|
|
98395
|
-
AND bundle_id = ?
|
|
98396
|
-
AND adapter_id = ?`).all(file, component.id, component.adapter);
|
|
98397
|
-
for (const row of rows)
|
|
98398
|
-
unindexableEntryIds.add(row.id);
|
|
98399
|
-
}
|
|
98400
|
-
for (const conceptId of rejectedConceptIds) {
|
|
98401
|
-
const itemRef = `${component.id}//${conceptId}`;
|
|
98402
|
-
const rows = db.prepare(`SELECT id FROM entries
|
|
98403
|
-
WHERE bundle_id = ? AND adapter_id = ?
|
|
98404
|
-
AND type = 'workflow'
|
|
98405
|
-
AND (concept_id = ? OR item_ref = ?)`).all(component.id, component.adapter, conceptId, itemRef);
|
|
98406
|
-
for (const row of rows)
|
|
98407
|
-
unindexableEntryIds.add(row.id);
|
|
98408
|
-
}
|
|
98409
|
-
deleteEntriesByIds(db, [...unindexableEntryIds]);
|
|
98410
|
-
mutated ||= unindexableEntryIds.size > 0;
|
|
98411
|
-
for (const { file, entry, conceptId, contentHash } of pairs) {
|
|
98412
|
-
let entryWithSize = entry;
|
|
98413
|
-
try {
|
|
98414
|
-
entryWithSize = { ...entry, fileSize: fs53.statSync(file).size };
|
|
98415
|
-
if (hasMarkdownFragmentContent(entry)) {
|
|
98416
|
-
setMarkdownFragmentContent(entryWithSize, getMarkdownFragmentContent(entry));
|
|
98417
|
-
}
|
|
98418
|
-
} catch {}
|
|
98419
|
-
const provenance = deriveEntryProvenance({ bundleId: component.id, componentId: component.id, adapterId: component.adapter }, entry.type, entry.name, conceptId);
|
|
98420
|
-
const supersededIds = db.prepare("SELECT id FROM entries WHERE file_path = ? AND item_ref <> ?").all(file, provenance.itemRef);
|
|
98421
|
-
deleteEntriesByIds(db, supersededIds.map((row) => row.id));
|
|
98422
|
-
targetEntryIds.add(upsertEntry(db, file, entryWithSize, buildSearchText(entry), provenance, contentHash));
|
|
98423
|
-
mutated = true;
|
|
98424
|
-
}
|
|
98425
|
-
})();
|
|
98426
|
-
if (mutated) {
|
|
98326
|
+
await reconcilePaths(db, files, bundleId, { root: stashDir });
|
|
98327
|
+
try {
|
|
98427
98328
|
const config = loadConfig();
|
|
98428
|
-
|
|
98429
|
-
|
|
98329
|
+
const onlyHashes = unitHashesForFiles(db, files);
|
|
98330
|
+
await drainEmbeddingQueue(db, config, { onlyHashes });
|
|
98331
|
+
} catch (drainError) {
|
|
98332
|
+
warnVerbose("Write-path embedding drain skipped (vectors appear after the next drain):", drainError instanceof Error ? drainError.message : String(drainError));
|
|
98430
98333
|
}
|
|
98334
|
+
return true;
|
|
98431
98335
|
} finally {
|
|
98432
98336
|
closeDatabase(db);
|
|
98433
98337
|
}
|
|
98434
|
-
return true;
|
|
98435
98338
|
})();
|
|
98436
98339
|
} catch (error2) {
|
|
98437
98340
|
if (isDataDirUnreadableError(error2)) {
|
|
@@ -98447,7 +98350,7 @@ async function indexWrittenAssets(stashDir, filePaths, options = {}) {
|
|
|
98447
98350
|
init_asset_placement();
|
|
98448
98351
|
init_asset_ref();
|
|
98449
98352
|
init_warn();
|
|
98450
|
-
import
|
|
98353
|
+
import path64 from "node:path";
|
|
98451
98354
|
function changesToStored(changes) {
|
|
98452
98355
|
return changes.map((c, i) => ({
|
|
98453
98356
|
path: c.path,
|
|
@@ -98500,10 +98403,10 @@ function currentProposalTarget(value) {
|
|
|
98500
98403
|
if (typeof value !== "object" || value === null)
|
|
98501
98404
|
throw new Error("Proposal metadata has an invalid proposedTarget.");
|
|
98502
98405
|
const target = value;
|
|
98503
|
-
if (typeof target.source !== "string" || !isBundleSlug(target.source) || typeof target.root !== "string" || !
|
|
98406
|
+
if (typeof target.source !== "string" || !isBundleSlug(target.source) || typeof target.root !== "string" || !path64.isAbsolute(target.root)) {
|
|
98504
98407
|
throw new Error("Proposal metadata has an invalid proposedTarget.");
|
|
98505
98408
|
}
|
|
98506
|
-
return { source: target.source, root:
|
|
98409
|
+
return { source: target.source, root: path64.resolve(target.root) };
|
|
98507
98410
|
}
|
|
98508
98411
|
function invalidPresentField(name) {
|
|
98509
98412
|
throw new Error(`Proposal metadata has an invalid ${name}.`);
|
|
@@ -98543,7 +98446,7 @@ function validatePresentMetadata(meta) {
|
|
|
98543
98446
|
}
|
|
98544
98447
|
if (Object.hasOwn(meta, "acceptedTarget")) {
|
|
98545
98448
|
const target = meta.acceptedTarget;
|
|
98546
|
-
if (typeof target !== "object" || target === null || typeof target.source !== "string" || !isBundleSlug(target.source) || typeof target.root !== "string" || !
|
|
98449
|
+
if (typeof target !== "object" || target === null || typeof target.source !== "string" || !isBundleSlug(target.source) || typeof target.root !== "string" || !path64.isAbsolute(target.root) || typeof target.path !== "string" || !path64.isAbsolute(target.path) || typeof target.contentHash !== "string") {
|
|
98547
98450
|
invalidPresentField("acceptedTarget");
|
|
98548
98451
|
}
|
|
98549
98452
|
}
|
|
@@ -99313,12 +99216,12 @@ function proposalHash(content) {
|
|
|
99313
99216
|
return createHash11("sha256").update(content).digest("hex");
|
|
99314
99217
|
}
|
|
99315
99218
|
function proposalFileHash(filePath) {
|
|
99316
|
-
return proposalHash(
|
|
99219
|
+
return proposalHash(fs53.readFileSync(filePath));
|
|
99317
99220
|
}
|
|
99318
99221
|
function sameProposalFile(left, right) {
|
|
99319
99222
|
try {
|
|
99320
|
-
const leftStat =
|
|
99321
|
-
const rightStat =
|
|
99223
|
+
const leftStat = fs53.statSync(left);
|
|
99224
|
+
const rightStat = fs53.statSync(right);
|
|
99322
99225
|
return leftStat.dev === rightStat.dev && leftStat.ino === rightStat.ino;
|
|
99323
99226
|
} catch {
|
|
99324
99227
|
return false;
|
|
@@ -99327,20 +99230,20 @@ function sameProposalFile(left, right) {
|
|
|
99327
99230
|
function cleanupProposalPublication(p) {
|
|
99328
99231
|
for (const filePath of [p.publishPath, p.displacedPath]) {
|
|
99329
99232
|
try {
|
|
99330
|
-
|
|
99233
|
+
fs53.rmSync(filePath, { force: true });
|
|
99331
99234
|
} catch (error2) {
|
|
99332
99235
|
warn(`[proposals] transaction publication cleanup failed at ${filePath}: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
99333
99236
|
}
|
|
99334
99237
|
}
|
|
99335
|
-
fsyncTxnDir(
|
|
99238
|
+
fsyncTxnDir(path65.dirname(p.assetPath));
|
|
99336
99239
|
}
|
|
99337
99240
|
function rollbackPreparedProposalTransaction(txn) {
|
|
99338
99241
|
const p = txn.journal.payload;
|
|
99339
|
-
const currentHash =
|
|
99340
|
-
if (!
|
|
99242
|
+
const currentHash = fs53.existsSync(p.assetPath) ? proposalFileHash(p.assetPath) : null;
|
|
99243
|
+
if (!fs53.existsSync(p.displacedPath)) {
|
|
99341
99244
|
if (p.originalHash === null) {
|
|
99342
99245
|
if (currentHash === p.publishedHash && sameProposalFile(p.assetPath, p.publishPath)) {
|
|
99343
|
-
|
|
99246
|
+
fs53.unlinkSync(p.assetPath);
|
|
99344
99247
|
recordWrittenPath(p.assetPath);
|
|
99345
99248
|
} else if (currentHash !== null) {
|
|
99346
99249
|
throw new Error(`Cannot roll back proposal transaction: target was created externally.`);
|
|
@@ -99352,30 +99255,30 @@ function rollbackPreparedProposalTransaction(txn) {
|
|
|
99352
99255
|
return;
|
|
99353
99256
|
}
|
|
99354
99257
|
if (currentHash === p.publishedHash) {
|
|
99355
|
-
|
|
99258
|
+
fs53.unlinkSync(p.assetPath);
|
|
99356
99259
|
recordWrittenPath(p.assetPath);
|
|
99357
99260
|
} else if (currentHash !== null && currentHash !== p.originalHash) {
|
|
99358
99261
|
throw new Error(`Cannot roll back proposal transaction: ${p.assetPath} diverged.`);
|
|
99359
99262
|
}
|
|
99360
|
-
if (
|
|
99361
|
-
if (
|
|
99263
|
+
if (fs53.existsSync(p.displacedPath)) {
|
|
99264
|
+
if (fs53.existsSync(p.assetPath)) {
|
|
99362
99265
|
throw new Error(`Cannot restore proposal backup: ${p.assetPath} is occupied.`);
|
|
99363
99266
|
}
|
|
99364
|
-
|
|
99267
|
+
fs53.linkSync(p.displacedPath, p.assetPath);
|
|
99365
99268
|
recordWrittenPath(p.assetPath);
|
|
99366
99269
|
}
|
|
99367
99270
|
cleanupProposalPublication(p);
|
|
99368
99271
|
}
|
|
99369
99272
|
function validatePublishedProposal(p) {
|
|
99370
|
-
if (!
|
|
99273
|
+
if (!fs53.existsSync(p.assetPath) || proposalFileHash(p.assetPath) !== p.publishedHash) {
|
|
99371
99274
|
throw new Error(`Cannot recover proposal ${p.proposalId}: published asset diverged.`);
|
|
99372
99275
|
}
|
|
99373
99276
|
}
|
|
99374
99277
|
function persistProposalTransactionState(txn, proposal, ctx) {
|
|
99375
99278
|
const p = txn.journal.payload;
|
|
99376
99279
|
const decidedAt = txn.journal.decidedAt;
|
|
99377
|
-
const backupContent = p.backupPath ?
|
|
99378
|
-
const publishedContent =
|
|
99280
|
+
const backupContent = p.backupPath ? fs53.readFileSync(p.backupPath, "utf8") : undefined;
|
|
99281
|
+
const publishedContent = fs53.readFileSync(p.contentPath, "utf8");
|
|
99379
99282
|
return withProposalsDb(p.stashDir, ctx, (db) => withImmediateTransaction(db, () => {
|
|
99380
99283
|
const current = requireProposal(db, p.stashDir, p.proposalId);
|
|
99381
99284
|
if (p.operation === "accept") {
|
|
@@ -99393,7 +99296,7 @@ function persistProposalTransactionState(txn, proposal, ctx) {
|
|
|
99393
99296
|
payload: { ...proposal.payload, content: publishedContent },
|
|
99394
99297
|
changes: [
|
|
99395
99298
|
{
|
|
99396
|
-
path:
|
|
99299
|
+
path: path65.relative(txn.journal.root, p.assetPath),
|
|
99397
99300
|
op: p.originalHash === null ? "create" : "update",
|
|
99398
99301
|
after: publishedContent
|
|
99399
99302
|
}
|
|
@@ -99464,7 +99367,7 @@ async function finalizeProposalTransaction(txn, target, proposal, ctx) {
|
|
|
99464
99367
|
cleanupProposalPublication(p);
|
|
99465
99368
|
if (txn.journal.phase === "asset-published") {
|
|
99466
99369
|
const commitRoot = target.source.repoPath ?? target.source.path;
|
|
99467
|
-
const commitPath =
|
|
99370
|
+
const commitPath = path65.relative(commitRoot, p.assetPath).replaceAll(path65.sep, "/");
|
|
99468
99371
|
publishWriteTargetTransaction(target, p.gitPublication, {
|
|
99469
99372
|
transactionId: txn.journal.transactionId,
|
|
99470
99373
|
message: `${p.operation === "accept" ? "Update" : "Revert"} ${p.ref}`,
|
|
@@ -99483,10 +99386,11 @@ async function finalizeProposalTransaction(txn, target, proposal, ctx) {
|
|
|
99483
99386
|
}
|
|
99484
99387
|
let accepted = getProposal(p.stashDir, p.proposalId, ctx);
|
|
99485
99388
|
if (txn.journal.phase === "proposal-persisted") {
|
|
99486
|
-
if (
|
|
99487
|
-
|
|
99389
|
+
if (await indexWrittenAssets(txn.journal.root, [p.assetPath], { bundleId: target.source.name })) {
|
|
99390
|
+
advanceTxn(txn, "index-finalized");
|
|
99391
|
+
} else {
|
|
99392
|
+
warn(`${p.operation === "accept" ? "Accept" : "Revert"} of ${p.ref} succeeded, but its index update failed; run \`akm index\` to refresh it.`);
|
|
99488
99393
|
}
|
|
99489
|
-
advanceTxn(txn, "index-finalized");
|
|
99490
99394
|
}
|
|
99491
99395
|
if (txn.journal.phase === "index-finalized") {
|
|
99492
99396
|
accepted = getProposal(p.stashDir, p.proposalId, ctx);
|
|
@@ -99501,8 +99405,8 @@ async function finalizeProposalTransaction(txn, target, proposal, ctx) {
|
|
|
99501
99405
|
function fenceProposalTxnJournal(journal, txnDir, root) {
|
|
99502
99406
|
const p = journal.payload;
|
|
99503
99407
|
const refIdentity = proposalRefIdentity(p.ref);
|
|
99504
|
-
if (!["accept", "revert"].includes(p.operation) || !p.targetSource || !p.targetKind || refIdentity?.bundle === undefined || !isWithin(p.assetPath, root) || ![p.contentPath, p.backupPath].filter((candidate) => candidate !== null).every((candidate) => isWithin(candidate, txnDir)) || ![p.publishPath, p.displacedPath].every((candidate) => isWithin(candidate, root) &&
|
|
99505
|
-
throw new Error(`Refusing unsafe proposal transaction journal at ${
|
|
99408
|
+
if (!["accept", "revert"].includes(p.operation) || !p.targetSource || !p.targetKind || refIdentity?.bundle === undefined || !isWithin(p.assetPath, root) || ![p.contentPath, p.backupPath].filter((candidate) => candidate !== null).every((candidate) => isWithin(candidate, txnDir)) || ![p.publishPath, p.displacedPath].every((candidate) => isWithin(candidate, root) && path65.dirname(candidate) === path65.dirname(p.assetPath))) {
|
|
99409
|
+
throw new Error(`Refusing unsafe proposal transaction journal at ${path65.join(txnDir, "journal.json")}.`);
|
|
99506
99410
|
}
|
|
99507
99411
|
}
|
|
99508
99412
|
function resolveProposalRecoveryTarget(config, journal) {
|
|
@@ -99602,8 +99506,8 @@ async function recoverStaleTxns(stashDir) {
|
|
|
99602
99506
|
}
|
|
99603
99507
|
|
|
99604
99508
|
// scripts/akm-migrate/migrate/writer-relocation.ts
|
|
99605
|
-
import
|
|
99606
|
-
import
|
|
99509
|
+
import fs55 from "node:fs";
|
|
99510
|
+
import path66 from "node:path";
|
|
99607
99511
|
init_paths();
|
|
99608
99512
|
function relocationSpecs(stashDir) {
|
|
99609
99513
|
return [
|
|
@@ -99623,7 +99527,7 @@ function mutexSiblingName(lockName) {
|
|
|
99623
99527
|
function fileCountIfExists(dir) {
|
|
99624
99528
|
let entries;
|
|
99625
99529
|
try {
|
|
99626
|
-
entries =
|
|
99530
|
+
entries = fs55.readdirSync(dir, { withFileTypes: true });
|
|
99627
99531
|
} catch {
|
|
99628
99532
|
return;
|
|
99629
99533
|
}
|
|
@@ -99631,7 +99535,7 @@ function fileCountIfExists(dir) {
|
|
|
99631
99535
|
}
|
|
99632
99536
|
function statFileIfExists(filePath) {
|
|
99633
99537
|
try {
|
|
99634
|
-
const stat =
|
|
99538
|
+
const stat = fs55.statSync(filePath);
|
|
99635
99539
|
return stat.isFile() ? stat : undefined;
|
|
99636
99540
|
} catch {
|
|
99637
99541
|
return;
|
|
@@ -99641,8 +99545,8 @@ function classifyLockArtifacts(akmDir) {
|
|
|
99641
99545
|
const removable = [];
|
|
99642
99546
|
const skipped = [];
|
|
99643
99547
|
for (const lockName of LOCK_NAMES) {
|
|
99644
|
-
const lockPath =
|
|
99645
|
-
const mutexPath =
|
|
99548
|
+
const lockPath = path66.join(akmDir, lockName);
|
|
99549
|
+
const mutexPath = path66.join(akmDir, mutexSiblingName(lockName));
|
|
99646
99550
|
const lockStat = statFileIfExists(lockPath);
|
|
99647
99551
|
const mutexStat = statFileIfExists(mutexPath);
|
|
99648
99552
|
if (!lockStat) {
|
|
@@ -99668,11 +99572,11 @@ function classifyLockArtifacts(akmDir) {
|
|
|
99668
99572
|
return { removable, skipped };
|
|
99669
99573
|
}
|
|
99670
99574
|
function findWriterRelocationEntries(stashDir) {
|
|
99671
|
-
const akmDir =
|
|
99575
|
+
const akmDir = path66.join(stashDir, ".akm");
|
|
99672
99576
|
const directories = [];
|
|
99673
99577
|
for (const spec of relocationSpecs(stashDir)) {
|
|
99674
99578
|
const relativeParts = Array.isArray(spec.oldRelative) ? spec.oldRelative : [spec.oldRelative];
|
|
99675
|
-
const oldPath =
|
|
99579
|
+
const oldPath = path66.join(akmDir, ...relativeParts);
|
|
99676
99580
|
const fileCount = fileCountIfExists(oldPath);
|
|
99677
99581
|
if (fileCount === undefined || fileCount === 0)
|
|
99678
99582
|
continue;
|
|
@@ -99683,36 +99587,36 @@ function findWriterRelocationEntries(stashDir) {
|
|
|
99683
99587
|
}
|
|
99684
99588
|
function moveFile(oldFilePath, newFilePath) {
|
|
99685
99589
|
try {
|
|
99686
|
-
|
|
99590
|
+
fs55.renameSync(oldFilePath, newFilePath);
|
|
99687
99591
|
} catch (error2) {
|
|
99688
99592
|
if (error2.code !== "EXDEV")
|
|
99689
99593
|
throw error2;
|
|
99690
|
-
|
|
99691
|
-
|
|
99594
|
+
fs55.copyFileSync(oldFilePath, newFilePath);
|
|
99595
|
+
fs55.rmSync(oldFilePath, { force: true });
|
|
99692
99596
|
}
|
|
99693
99597
|
}
|
|
99694
99598
|
function moveDirectoryContents(entry) {
|
|
99695
99599
|
const errors3 = [];
|
|
99696
99600
|
let moved = 0;
|
|
99697
|
-
|
|
99601
|
+
fs55.mkdirSync(entry.newPath, { recursive: true });
|
|
99698
99602
|
let names;
|
|
99699
99603
|
try {
|
|
99700
|
-
names =
|
|
99604
|
+
names = fs55.readdirSync(entry.oldPath).sort();
|
|
99701
99605
|
} catch {
|
|
99702
99606
|
return { key: entry.key, oldPath: entry.oldPath, newPath: entry.newPath, moved: 0, errors: [] };
|
|
99703
99607
|
}
|
|
99704
99608
|
for (const name of names) {
|
|
99705
|
-
const oldFilePath =
|
|
99706
|
-
const newFilePath =
|
|
99609
|
+
const oldFilePath = path66.join(entry.oldPath, name);
|
|
99610
|
+
const newFilePath = path66.join(entry.newPath, name);
|
|
99707
99611
|
let oldStat;
|
|
99708
99612
|
try {
|
|
99709
|
-
oldStat =
|
|
99613
|
+
oldStat = fs55.lstatSync(oldFilePath);
|
|
99710
99614
|
} catch {
|
|
99711
99615
|
continue;
|
|
99712
99616
|
}
|
|
99713
99617
|
if (!oldStat.isFile())
|
|
99714
99618
|
continue;
|
|
99715
|
-
if (
|
|
99619
|
+
if (fs55.existsSync(newFilePath))
|
|
99716
99620
|
continue;
|
|
99717
99621
|
try {
|
|
99718
99622
|
moveFile(oldFilePath, newFilePath);
|
|
@@ -99725,13 +99629,13 @@ function moveDirectoryContents(entry) {
|
|
|
99725
99629
|
}
|
|
99726
99630
|
function removeIfEmptyDir(dir) {
|
|
99727
99631
|
try {
|
|
99728
|
-
if (
|
|
99729
|
-
|
|
99632
|
+
if (fs55.readdirSync(dir).length === 0)
|
|
99633
|
+
fs55.rmdirSync(dir);
|
|
99730
99634
|
} catch {}
|
|
99731
99635
|
}
|
|
99732
99636
|
function removeLockArtifact(entry) {
|
|
99733
99637
|
try {
|
|
99734
|
-
|
|
99638
|
+
fs55.rmSync(entry.path, { force: true });
|
|
99735
99639
|
return { path: entry.path, removed: true };
|
|
99736
99640
|
} catch (error2) {
|
|
99737
99641
|
return { path: entry.path, removed: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
@@ -99743,36 +99647,36 @@ function applyWriterRelocation(stashDir) {
|
|
|
99743
99647
|
const lockResults = lockArtifacts.map(removeLockArtifact);
|
|
99744
99648
|
for (const spec of relocationSpecs(stashDir)) {
|
|
99745
99649
|
const relativeParts = Array.isArray(spec.oldRelative) ? spec.oldRelative : [spec.oldRelative];
|
|
99746
|
-
removeIfEmptyDir(
|
|
99650
|
+
removeIfEmptyDir(path66.join(stashDir, ".akm", ...relativeParts));
|
|
99747
99651
|
}
|
|
99748
99652
|
return { directories: directoryResults, lockArtifacts: lockResults, skippedLocks };
|
|
99749
99653
|
}
|
|
99750
99654
|
|
|
99751
99655
|
// scripts/akm-migrate/task-migrate.ts
|
|
99752
99656
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
99753
|
-
import
|
|
99657
|
+
import fs59 from "node:fs";
|
|
99754
99658
|
import os9 from "node:os";
|
|
99755
|
-
import
|
|
99659
|
+
import path69 from "node:path";
|
|
99756
99660
|
init_errors();
|
|
99757
99661
|
init_paths();
|
|
99758
99662
|
|
|
99759
99663
|
// scripts/akm-migrate/migrate/task-files-to-v3.ts
|
|
99760
99664
|
init_errors();
|
|
99761
99665
|
import crypto6 from "node:crypto";
|
|
99762
|
-
import
|
|
99763
|
-
import
|
|
99666
|
+
import fs57 from "node:fs";
|
|
99667
|
+
import path67 from "node:path";
|
|
99764
99668
|
|
|
99765
99669
|
// scripts/akm-migrate/migrate/durable-fs.ts
|
|
99766
|
-
import
|
|
99670
|
+
import fs56 from "node:fs";
|
|
99767
99671
|
function fsyncDirectoryPortable(directory) {
|
|
99768
99672
|
if (process.platform === "win32")
|
|
99769
99673
|
return;
|
|
99770
99674
|
try {
|
|
99771
|
-
const fd =
|
|
99675
|
+
const fd = fs56.openSync(directory, "r");
|
|
99772
99676
|
try {
|
|
99773
|
-
|
|
99677
|
+
fs56.fsyncSync(fd);
|
|
99774
99678
|
} finally {
|
|
99775
|
-
|
|
99679
|
+
fs56.closeSync(fd);
|
|
99776
99680
|
}
|
|
99777
99681
|
} catch (cause) {
|
|
99778
99682
|
const code = cause.code;
|
|
@@ -99786,25 +99690,25 @@ function migrationError(detail) {
|
|
|
99786
99690
|
return new ConfigError(`Task migration to v3 failed: ${detail}`, "INVALID_CONFIG_FILE");
|
|
99787
99691
|
}
|
|
99788
99692
|
function contained2(root, candidate) {
|
|
99789
|
-
const relative =
|
|
99790
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
99693
|
+
const relative = path67.relative(root, candidate);
|
|
99694
|
+
return relative === "" || !relative.startsWith("..") && !path67.isAbsolute(relative);
|
|
99791
99695
|
}
|
|
99792
99696
|
function realDirectory(filePath) {
|
|
99793
|
-
const stat =
|
|
99697
|
+
const stat = fs57.lstatSync(filePath);
|
|
99794
99698
|
if (stat.isSymbolicLink() || !stat.isDirectory())
|
|
99795
99699
|
throw migrationError(`${filePath} must be a real directory.`);
|
|
99796
|
-
return
|
|
99700
|
+
return fs57.realpathSync(filePath);
|
|
99797
99701
|
}
|
|
99798
99702
|
function snapshot(filePath) {
|
|
99799
|
-
const stat =
|
|
99703
|
+
const stat = fs57.lstatSync(filePath);
|
|
99800
99704
|
if (stat.isSymbolicLink() || !stat.isFile())
|
|
99801
99705
|
throw migrationError(`${filePath} must be a real file.`);
|
|
99802
|
-
const bytes =
|
|
99706
|
+
const bytes = fs57.readFileSync(filePath);
|
|
99803
99707
|
return Object.freeze({ bytes, mode: stat.mode & 511 });
|
|
99804
99708
|
}
|
|
99805
99709
|
function writable(filePath) {
|
|
99806
99710
|
try {
|
|
99807
|
-
|
|
99711
|
+
fs57.accessSync(filePath, fs57.constants.W_OK);
|
|
99808
99712
|
return true;
|
|
99809
99713
|
} catch {
|
|
99810
99714
|
return false;
|
|
@@ -99817,12 +99721,12 @@ function walkTasks(root, tasksDir, out) {
|
|
|
99817
99721
|
throw migrationError(`${root.root} resolves outside bundle ${root.bundleId}.`);
|
|
99818
99722
|
}
|
|
99819
99723
|
const visit2 = (directory) => {
|
|
99820
|
-
const physicalDirectory =
|
|
99724
|
+
const physicalDirectory = fs57.realpathSync(directory);
|
|
99821
99725
|
if (!contained2(physicalRoot, physicalDirectory)) {
|
|
99822
99726
|
throw migrationError(`${directory} resolves outside bundle ${root.bundleId}.`);
|
|
99823
99727
|
}
|
|
99824
|
-
for (const entry of
|
|
99825
|
-
const candidate =
|
|
99728
|
+
for (const entry of fs57.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
99729
|
+
const candidate = path67.join(directory, entry.name);
|
|
99826
99730
|
if (entry.isSymbolicLink())
|
|
99827
99731
|
throw migrationError(`task migration does not follow symbolic link ${candidate}.`);
|
|
99828
99732
|
if (entry.isDirectory()) {
|
|
@@ -99832,7 +99736,7 @@ function walkTasks(root, tasksDir, out) {
|
|
|
99832
99736
|
if (!entry.isFile() || !entry.name.endsWith(".yml"))
|
|
99833
99737
|
continue;
|
|
99834
99738
|
const current = snapshot(candidate);
|
|
99835
|
-
const parent =
|
|
99739
|
+
const parent = path67.dirname(candidate);
|
|
99836
99740
|
out.push({
|
|
99837
99741
|
filePath: candidate,
|
|
99838
99742
|
bytes: current.bytes,
|
|
@@ -99848,9 +99752,9 @@ function walkTasks(root, tasksDir, out) {
|
|
|
99848
99752
|
function inspectTaskToV3Files(roots) {
|
|
99849
99753
|
const files = [];
|
|
99850
99754
|
for (const root of [...roots].sort((a, b) => a.bundleId.localeCompare(b.bundleId))) {
|
|
99851
|
-
const tasksDir = root.layout === "akm-task" ? root.root :
|
|
99755
|
+
const tasksDir = root.layout === "akm-task" ? root.root : path67.join(root.root, "tasks");
|
|
99852
99756
|
try {
|
|
99853
|
-
const stat =
|
|
99757
|
+
const stat = fs57.lstatSync(tasksDir);
|
|
99854
99758
|
if (stat.isSymbolicLink())
|
|
99855
99759
|
throw migrationError(`task migration does not follow symbolic link ${tasksDir}.`);
|
|
99856
99760
|
if (!stat.isDirectory())
|
|
@@ -99865,33 +99769,33 @@ function inspectTaskToV3Files(roots) {
|
|
|
99865
99769
|
return files.sort((a, b) => a.filePath.localeCompare(b.filePath));
|
|
99866
99770
|
}
|
|
99867
99771
|
function hashPath(filePath) {
|
|
99868
|
-
return crypto6.createHash("sha256").update(
|
|
99772
|
+
return crypto6.createHash("sha256").update(path67.resolve(filePath)).digest("hex").slice(0, 16);
|
|
99869
99773
|
}
|
|
99870
99774
|
function taskMigrationBackupPath(backupRoot, filePath) {
|
|
99871
|
-
return
|
|
99775
|
+
return path67.join(backupRoot, "files", `${hashPath(filePath)}-${path67.basename(filePath)}`);
|
|
99872
99776
|
}
|
|
99873
99777
|
function writeDurable(filePath, bytes, mode, exclusive = false) {
|
|
99874
|
-
|
|
99778
|
+
fs57.mkdirSync(path67.dirname(filePath), { recursive: true });
|
|
99875
99779
|
const flags = exclusive ? "wx" : "w";
|
|
99876
|
-
const fd =
|
|
99780
|
+
const fd = fs57.openSync(filePath, flags, mode);
|
|
99877
99781
|
try {
|
|
99878
|
-
|
|
99879
|
-
|
|
99782
|
+
fs57.writeFileSync(fd, bytes);
|
|
99783
|
+
fs57.fsyncSync(fd);
|
|
99880
99784
|
} finally {
|
|
99881
|
-
|
|
99785
|
+
fs57.closeSync(fd);
|
|
99882
99786
|
}
|
|
99883
|
-
|
|
99884
|
-
fsyncDirectoryPortable(
|
|
99787
|
+
fs57.chmodSync(filePath, mode);
|
|
99788
|
+
fsyncDirectoryPortable(path67.dirname(filePath));
|
|
99885
99789
|
}
|
|
99886
99790
|
function replaceAtomically(filePath, bytes, mode) {
|
|
99887
|
-
const temporary =
|
|
99791
|
+
const temporary = path67.join(path67.dirname(filePath), `.${path67.basename(filePath)}.migrate-${crypto6.randomUUID()}`);
|
|
99888
99792
|
try {
|
|
99889
99793
|
writeDurable(temporary, bytes, mode, true);
|
|
99890
|
-
|
|
99891
|
-
fsyncDirectoryPortable(
|
|
99794
|
+
fs57.renameSync(temporary, filePath);
|
|
99795
|
+
fsyncDirectoryPortable(path67.dirname(filePath));
|
|
99892
99796
|
} finally {
|
|
99893
99797
|
try {
|
|
99894
|
-
|
|
99798
|
+
fs57.unlinkSync(temporary);
|
|
99895
99799
|
} catch (cause) {
|
|
99896
99800
|
if (cause.code !== "ENOENT")
|
|
99897
99801
|
throw cause;
|
|
@@ -99930,7 +99834,7 @@ function applyTaskToV3MigrationPlan(plan, options) {
|
|
|
99930
99834
|
const current = snapshot(change.filePath);
|
|
99931
99835
|
if (!current.bytes.equals(change.after))
|
|
99932
99836
|
continue;
|
|
99933
|
-
replaceAtomically(change.filePath,
|
|
99837
|
+
replaceAtomically(change.filePath, fs57.readFileSync(taskMigrationBackupPath(options.backupRoot, change.filePath)), change.mode);
|
|
99934
99838
|
}
|
|
99935
99839
|
throw cause;
|
|
99936
99840
|
}
|
|
@@ -99940,31 +99844,31 @@ function applyTaskToV3MigrationPlan(plan, options) {
|
|
|
99940
99844
|
// scripts/akm-migrate/migrate/task-files-to-v4.ts
|
|
99941
99845
|
init_errors();
|
|
99942
99846
|
import crypto7 from "node:crypto";
|
|
99943
|
-
import
|
|
99944
|
-
import
|
|
99847
|
+
import fs58 from "node:fs";
|
|
99848
|
+
import path68 from "node:path";
|
|
99945
99849
|
function migrationError2(detail) {
|
|
99946
99850
|
return new ConfigError(`Task migration to v4 failed: ${detail}`, "INVALID_CONFIG_FILE");
|
|
99947
99851
|
}
|
|
99948
99852
|
function contained3(root, candidate) {
|
|
99949
|
-
const relative =
|
|
99950
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
99853
|
+
const relative = path68.relative(root, candidate);
|
|
99854
|
+
return relative === "" || !relative.startsWith("..") && !path68.isAbsolute(relative);
|
|
99951
99855
|
}
|
|
99952
99856
|
function realDirectory2(filePath) {
|
|
99953
|
-
const stat =
|
|
99857
|
+
const stat = fs58.lstatSync(filePath);
|
|
99954
99858
|
if (stat.isSymbolicLink() || !stat.isDirectory())
|
|
99955
99859
|
throw migrationError2(`${filePath} must be a real directory.`);
|
|
99956
|
-
return
|
|
99860
|
+
return fs58.realpathSync(filePath);
|
|
99957
99861
|
}
|
|
99958
99862
|
function snapshot2(filePath) {
|
|
99959
|
-
const stat =
|
|
99863
|
+
const stat = fs58.lstatSync(filePath);
|
|
99960
99864
|
if (stat.isSymbolicLink() || !stat.isFile())
|
|
99961
99865
|
throw migrationError2(`${filePath} must be a real file.`);
|
|
99962
|
-
const bytes =
|
|
99866
|
+
const bytes = fs58.readFileSync(filePath);
|
|
99963
99867
|
return Object.freeze({ bytes, mode: stat.mode & 511 });
|
|
99964
99868
|
}
|
|
99965
99869
|
function writable2(filePath) {
|
|
99966
99870
|
try {
|
|
99967
|
-
|
|
99871
|
+
fs58.accessSync(filePath, fs58.constants.W_OK);
|
|
99968
99872
|
return true;
|
|
99969
99873
|
} catch {
|
|
99970
99874
|
return false;
|
|
@@ -99977,12 +99881,12 @@ function walkTasks2(root, tasksDir, out) {
|
|
|
99977
99881
|
throw migrationError2(`${root.root} resolves outside bundle ${root.bundleId}.`);
|
|
99978
99882
|
}
|
|
99979
99883
|
const visit2 = (directory) => {
|
|
99980
|
-
const physicalDirectory =
|
|
99884
|
+
const physicalDirectory = fs58.realpathSync(directory);
|
|
99981
99885
|
if (!contained3(physicalRoot, physicalDirectory)) {
|
|
99982
99886
|
throw migrationError2(`${directory} resolves outside bundle ${root.bundleId}.`);
|
|
99983
99887
|
}
|
|
99984
|
-
for (const entry of
|
|
99985
|
-
const candidate =
|
|
99888
|
+
for (const entry of fs58.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
99889
|
+
const candidate = path68.join(directory, entry.name);
|
|
99986
99890
|
if (entry.isSymbolicLink())
|
|
99987
99891
|
throw migrationError2(`task migration does not follow symbolic link ${candidate}.`);
|
|
99988
99892
|
if (entry.isDirectory()) {
|
|
@@ -99992,7 +99896,7 @@ function walkTasks2(root, tasksDir, out) {
|
|
|
99992
99896
|
if (!entry.isFile() || !entry.name.endsWith(".yml"))
|
|
99993
99897
|
continue;
|
|
99994
99898
|
const current = snapshot2(candidate);
|
|
99995
|
-
const parent =
|
|
99899
|
+
const parent = path68.dirname(candidate);
|
|
99996
99900
|
out.push({
|
|
99997
99901
|
filePath: candidate,
|
|
99998
99902
|
bytes: current.bytes,
|
|
@@ -100008,9 +99912,9 @@ function walkTasks2(root, tasksDir, out) {
|
|
|
100008
99912
|
function inspectTaskToV4Files(roots) {
|
|
100009
99913
|
const files = [];
|
|
100010
99914
|
for (const root of [...roots].sort((a, b) => a.bundleId.localeCompare(b.bundleId))) {
|
|
100011
|
-
const tasksDir = root.layout === "akm-task" ? root.root :
|
|
99915
|
+
const tasksDir = root.layout === "akm-task" ? root.root : path68.join(root.root, "tasks");
|
|
100012
99916
|
try {
|
|
100013
|
-
const stat =
|
|
99917
|
+
const stat = fs58.lstatSync(tasksDir);
|
|
100014
99918
|
if (stat.isSymbolicLink())
|
|
100015
99919
|
throw migrationError2(`task migration does not follow symbolic link ${tasksDir}.`);
|
|
100016
99920
|
if (!stat.isDirectory())
|
|
@@ -100025,33 +99929,33 @@ function inspectTaskToV4Files(roots) {
|
|
|
100025
99929
|
return files.sort((a, b) => a.filePath.localeCompare(b.filePath));
|
|
100026
99930
|
}
|
|
100027
99931
|
function hashPath2(filePath) {
|
|
100028
|
-
return crypto7.createHash("sha256").update(
|
|
99932
|
+
return crypto7.createHash("sha256").update(path68.resolve(filePath)).digest("hex").slice(0, 16);
|
|
100029
99933
|
}
|
|
100030
99934
|
function taskMigrationBackupPathV4(backupRoot, filePath) {
|
|
100031
|
-
return
|
|
99935
|
+
return path68.join(backupRoot, "files", `${hashPath2(filePath)}-${path68.basename(filePath)}`);
|
|
100032
99936
|
}
|
|
100033
99937
|
function writeDurable2(filePath, bytes, mode, exclusive = false) {
|
|
100034
|
-
|
|
99938
|
+
fs58.mkdirSync(path68.dirname(filePath), { recursive: true });
|
|
100035
99939
|
const flags = exclusive ? "wx" : "w";
|
|
100036
|
-
const fd =
|
|
99940
|
+
const fd = fs58.openSync(filePath, flags, mode);
|
|
100037
99941
|
try {
|
|
100038
|
-
|
|
100039
|
-
|
|
99942
|
+
fs58.writeFileSync(fd, bytes);
|
|
99943
|
+
fs58.fsyncSync(fd);
|
|
100040
99944
|
} finally {
|
|
100041
|
-
|
|
99945
|
+
fs58.closeSync(fd);
|
|
100042
99946
|
}
|
|
100043
|
-
|
|
100044
|
-
fsyncDirectoryPortable(
|
|
99947
|
+
fs58.chmodSync(filePath, mode);
|
|
99948
|
+
fsyncDirectoryPortable(path68.dirname(filePath));
|
|
100045
99949
|
}
|
|
100046
99950
|
function replaceAtomically2(filePath, bytes, mode) {
|
|
100047
|
-
const temporary =
|
|
99951
|
+
const temporary = path68.join(path68.dirname(filePath), `.${path68.basename(filePath)}.migrate-${crypto7.randomUUID()}`);
|
|
100048
99952
|
try {
|
|
100049
99953
|
writeDurable2(temporary, bytes, mode, true);
|
|
100050
|
-
|
|
100051
|
-
fsyncDirectoryPortable(
|
|
99954
|
+
fs58.renameSync(temporary, filePath);
|
|
99955
|
+
fsyncDirectoryPortable(path68.dirname(filePath));
|
|
100052
99956
|
} finally {
|
|
100053
99957
|
try {
|
|
100054
|
-
|
|
99958
|
+
fs58.unlinkSync(temporary);
|
|
100055
99959
|
} catch (cause) {
|
|
100056
99960
|
if (cause.code !== "ENOENT")
|
|
100057
99961
|
throw cause;
|
|
@@ -100090,7 +99994,7 @@ function applyTaskToV4MigrationPlan(plan, options) {
|
|
|
100090
99994
|
const current = snapshot2(change.filePath);
|
|
100091
99995
|
if (!current.bytes.equals(change.after))
|
|
100092
99996
|
continue;
|
|
100093
|
-
replaceAtomically2(change.filePath,
|
|
99997
|
+
replaceAtomically2(change.filePath, fs58.readFileSync(taskMigrationBackupPathV4(options.backupRoot, change.filePath)), change.mode);
|
|
100094
99998
|
}
|
|
100095
99999
|
throw cause;
|
|
100096
100000
|
}
|
|
@@ -100102,12 +100006,12 @@ function expandTilde(value) {
|
|
|
100102
100006
|
if (value === "~")
|
|
100103
100007
|
return os9.homedir();
|
|
100104
100008
|
if (value.startsWith("~/") || value.startsWith("~\\"))
|
|
100105
|
-
return
|
|
100009
|
+
return path69.join(os9.homedir(), value.slice(2));
|
|
100106
100010
|
return value;
|
|
100107
100011
|
}
|
|
100108
100012
|
function existingDirectory(target) {
|
|
100109
100013
|
try {
|
|
100110
|
-
return
|
|
100014
|
+
return fs59.statSync(target).isDirectory();
|
|
100111
100015
|
} catch (cause) {
|
|
100112
100016
|
if (cause.code === "ENOENT")
|
|
100113
100017
|
return false;
|
|
@@ -100128,21 +100032,21 @@ function taskRoots(config, resolutionBase = process.cwd()) {
|
|
|
100128
100032
|
const source = sources.get(bundleId);
|
|
100129
100033
|
if (!source)
|
|
100130
100034
|
continue;
|
|
100131
|
-
const configuredRoot = source.type === "filesystem" && source.path ?
|
|
100035
|
+
const configuredRoot = source.type === "filesystem" && source.path ? path69.resolve(resolutionBase, expandTilde(source.path)) : lockContentRootFor(bundleId, source.type);
|
|
100132
100036
|
if (!configuredRoot || !existingDirectory(configuredRoot))
|
|
100133
100037
|
continue;
|
|
100134
|
-
const bundleRoot =
|
|
100038
|
+
const bundleRoot = path69.resolve(configuredRoot);
|
|
100135
100039
|
const component = bundleComponentConfig(bundle);
|
|
100136
|
-
const componentRoot =
|
|
100137
|
-
const relative =
|
|
100138
|
-
if (relative === ".." || relative.startsWith(`..${
|
|
100040
|
+
const componentRoot = path69.resolve(bundleRoot, component?.root ?? ".");
|
|
100041
|
+
const relative = path69.relative(bundleRoot, componentRoot);
|
|
100042
|
+
if (relative === ".." || relative.startsWith(`..${path69.sep}`) || path69.isAbsolute(relative)) {
|
|
100139
100043
|
throw new ConfigError(`Task migration component root ${componentRoot} escapes bundle ${bundleId} at ${bundleRoot}.`, "INVALID_CONFIG_FILE");
|
|
100140
100044
|
}
|
|
100141
100045
|
if (!existingDirectory(componentRoot))
|
|
100142
100046
|
continue;
|
|
100143
100047
|
const adapter = component?.adapter ?? detectAdapterId(componentRoot, "");
|
|
100144
100048
|
if (!component?.adapter && adapter === "") {
|
|
100145
|
-
const flatTasks =
|
|
100049
|
+
const flatTasks = fs59.readdirSync(componentRoot, { withFileTypes: true }).filter((entry) => entry.isFile() && entry.name.endsWith(".yml")).map((entry) => entry.name).sort();
|
|
100146
100050
|
if (flatTasks.length > 0) {
|
|
100147
100051
|
throw new ConfigError(`Task migration cannot classify top-level task file(s) ${flatTasks.join(", ")} in bundle ${bundleId}; configure adapter "akm-task" or move them under tasks/.`, "INVALID_CONFIG_FILE");
|
|
100148
100052
|
}
|
|
@@ -100214,8 +100118,8 @@ function applyTaskV3Migration() {
|
|
|
100214
100118
|
const before = inspectCurrentTaskPlan();
|
|
100215
100119
|
if (before.result.taskV3Migration.changed === 0)
|
|
100216
100120
|
return before.result;
|
|
100217
|
-
const backupRoot =
|
|
100218
|
-
const backupPath =
|
|
100121
|
+
const backupRoot = path69.join(getDataDir(), "backups", "task-v3");
|
|
100122
|
+
const backupPath = path69.join(backupRoot, `${Date.now()}-${randomUUID7()}`);
|
|
100219
100123
|
const applied = applyTaskToV3MigrationPlan(before.plan, { backupRoot: backupPath });
|
|
100220
100124
|
const after = inspectCurrentTaskPlan().result;
|
|
100221
100125
|
if (after.taskV3Migration.changed > 0) {
|
|
@@ -100271,8 +100175,8 @@ function applyTaskV4Migration() {
|
|
|
100271
100175
|
const before = inspectCurrentTaskV4Plan();
|
|
100272
100176
|
if (before.result.taskV4Migration.changed === 0)
|
|
100273
100177
|
return before.result;
|
|
100274
|
-
const backupRoot =
|
|
100275
|
-
const backupPath =
|
|
100178
|
+
const backupRoot = path69.join(getDataDir(), "backups", "task-v4");
|
|
100179
|
+
const backupPath = path69.join(backupRoot, `${Date.now()}-${randomUUID7()}`);
|
|
100276
100180
|
const applied = applyTaskToV4MigrationPlan(before.plan, { backupRoot: backupPath });
|
|
100277
100181
|
const after = inspectCurrentTaskV4Plan().result;
|
|
100278
100182
|
if (after.taskV4Migration.changed > 0) {
|