akm-cli 0.9.15 → 0.9.16-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +144 -0
- package/dist/assets/tasks/core/index-refresh.yml +1 -1
- package/dist/cli/retired-commands.js +2 -0
- package/dist/cli/unknown-flags.js +36 -3
- package/dist/commands/improve/collapse-detector.js +2 -2
- package/dist/commands/improve/consolidate.js +6 -4
- package/dist/commands/improve/improve-cli.js +1 -1
- package/dist/commands/proposal/repository.js +12 -3
- package/dist/commands/read/curate.js +34 -44
- package/dist/commands/read/search.js +50 -2
- package/dist/commands/sources/index-status.js +99 -0
- package/dist/commands/sources/info.js +8 -8
- package/dist/commands/sources/installed-stashes.js +33 -12
- package/dist/commands/sources/source-add.js +21 -6
- package/dist/commands/sources/stash-cli.js +119 -111
- package/dist/core/adapter/adapters/akm-adapter.js +35 -3
- package/dist/core/adapter/adapters/akm-metadata.js +11 -1
- package/dist/core/asset/asset-placement.js +35 -0
- package/dist/core/config/schema/embedding.js +7 -30
- package/dist/core/config/schema/search.js +11 -9
- package/dist/core/errors.js +5 -2
- package/dist/core/hash.js +18 -0
- package/dist/core/maintenance-barrier.js +8 -6
- package/dist/core/paths.js +0 -11
- package/dist/core/run-lock.js +5 -2
- package/dist/core/state/migrations.js +26 -1
- package/dist/core/state-db.js +63 -27
- package/dist/indexer/drain.js +306 -0
- package/dist/indexer/embedding-identity.js +20 -0
- package/dist/indexer/enrich.js +260 -0
- package/dist/indexer/ensure-index.js +5 -0
- package/dist/indexer/index-written-assets.js +133 -171
- package/dist/indexer/indexer.js +458 -1621
- package/dist/indexer/lookup/adapter-concept-owner.js +19 -5
- package/dist/indexer/passes/metadata.js +18 -1
- package/dist/indexer/reconcile.js +890 -0
- package/dist/indexer/scan/drain-dir.js +27 -70
- package/dist/indexer/scan/parse-file.js +66 -0
- package/dist/indexer/search/db-search.js +373 -89
- package/dist/indexer/search/ranking-contributors.js +21 -16
- package/dist/indexer/search/ranking.js +135 -57
- package/dist/indexer/units/unit.js +159 -0
- package/dist/llm/client.js +10 -1
- package/dist/llm/embedder.js +10 -3
- package/dist/llm/embedders/provider-limits.js +288 -0
- package/dist/llm/embedders/remote.js +133 -104
- package/dist/llm/feature-gate.js +4 -2
- package/dist/llm/rerank-client.js +3 -3
- package/dist/output/shapes/passthrough.js +1 -0
- package/dist/output/text/command-format.js +19 -13
- package/dist/output/text/helpers.js +1 -1
- package/dist/output/text/index.js +5 -2
- package/dist/scripts/akm-migrate-node.js +1141 -1237
- package/dist/scripts/akm-migrate.js +1141 -1237
- package/dist/setup/semantic-assets.js +2 -2
- package/dist/setup/steps/connection.js +3 -2
- package/dist/storage/repositories/files-repository.js +181 -0
- package/dist/storage/repositories/index-connection.js +1 -3
- package/dist/storage/repositories/index-entries-repository.js +77 -68
- package/dist/storage/repositories/index-entry-schema.js +16 -25
- package/dist/storage/repositories/index-fts-repository.js +29 -263
- package/dist/storage/repositories/index-meta-repository.js +0 -29
- package/dist/storage/repositories/index-schema.js +115 -122
- package/dist/storage/repositories/index-utility-repository.js +1 -1
- package/dist/storage/repositories/index-vec-repository.js +21 -334
- package/dist/storage/repositories/units-repository.js +510 -0
- package/docs/migration/release-notes/0.9.15.md +34 -36
- package/docs/migration/release-notes/0.9.16.md +110 -0
- package/docs/migration/release-notes/README.md +5 -0
- package/docs/reference/cli.md +93 -87
- package/docs/reference/configuration.md +128 -89
- package/docs/reference/data-and-telemetry.md +2 -1
- package/package.json +1 -1
- package/schemas/akm-config.json +2 -58
- package/dist/indexer/index-db-contention.js +0 -56
- package/dist/indexer/index-rebuild-lock.js +0 -73
- package/dist/indexer/materialize-embeddings.js +0 -771
- package/dist/indexer/passes/dir-staleness.js +0 -161
- package/dist/storage/repositories/embedding-salvage-repository.js +0 -184
|
@@ -7043,7 +7043,8 @@ var init_errors = __esm(() => {
|
|
|
7043
7043
|
UNSAFE_STASH_DIR: "Choose a path inside your home directory (e.g. ~/akm) or another empty workspace. The bundle directory cannot be the filesystem root, your home directory itself, or a sensitive system path like /etc, /var, ~/.config, or ~/.ssh.",
|
|
7044
7044
|
UNKNOWN_IMPROVE_STRATEGY: "Pass one of the listed strategy names to `--strategy`, or define it under `improve.strategies`. Names are case-sensitive.",
|
|
7045
7045
|
EXECUTION_NOT_AUTHORIZED: "Change the selected tools or update the machine/user execution policy, then retry.",
|
|
7046
|
-
SECRET_REFERENCE_UNRESOLVED: "Check the secret exists (`akm secret list`) and the name after `secret://` matches, or run `akm secret set <name> <value>` to store it."
|
|
7046
|
+
SECRET_REFERENCE_UNRESOLVED: "Check the secret exists (`akm secret list`) and the name after `secret://` matches, or run `akm secret set <name> <value>` to store it.",
|
|
7047
|
+
EMBEDDING_VEC_UNAVAILABLE: "Install sqlite-vec for unit-level semantic search, or rely on lexical search until it is available."
|
|
7047
7048
|
};
|
|
7048
7049
|
COMPOSITION_INVALID_MULTI_JOB_HINT = "AKM workflows support exactly one job per source, with no needs: between jobs. Split the extra job(s) into " + "their own workflow file, and compose them with uses: workflows/<ref> instead.";
|
|
7049
7050
|
USAGE_HINTS = {
|
|
@@ -7070,8 +7071,8 @@ var init_errors = __esm(() => {
|
|
|
7070
7071
|
TRANSIENT_HINTS = {
|
|
7071
7072
|
RUN_LEASE_HELD: "Wait for the named engine invocation to finish or for the lease to expire, then retry. `akm workflow status <id>` shows the current lease.",
|
|
7072
7073
|
STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing.",
|
|
7073
|
-
INDEX_DB_CONTENDED: "Another akm process is writing index.db
|
|
7074
|
-
MAINTENANCE_BARRIER_BUSY: "Another akm process is registering a lock or lease right now. Retry shortly, or pass --skip-if-locked on scheduled
|
|
7074
|
+
INDEX_DB_CONTENDED: "Another akm process is writing index.db right now. Wait a few seconds and retry \u2014 index runs take no rebuild " + "lock, so this clears quickly; a scheduled run left alone will simply run again next time.",
|
|
7075
|
+
MAINTENANCE_BARRIER_BUSY: "Another akm process is registering a lock or lease right now. Retry shortly, or pass --skip-if-locked on " + "scheduled improve runs to skip gracefully instead \u2014 workflow run does not treat this code as skippable.",
|
|
7075
7076
|
IMPROVE_LOCK_HELD: "Another akm improve run holds the whole-run lock right now. Wait for it to finish and retry, or pass --skip-if-locked on scheduled runs."
|
|
7076
7077
|
};
|
|
7077
7078
|
NOT_FOUND_HINTS = {
|
|
@@ -7539,9 +7540,6 @@ function getDataDir(env = process.env, platform = process.platform) {
|
|
|
7539
7540
|
function getDbPath(env = process.env) {
|
|
7540
7541
|
return path3.join(getDataDir(env), "index.db");
|
|
7541
7542
|
}
|
|
7542
|
-
function getIndexRebuildLockPath() {
|
|
7543
|
-
return path3.join(getDataDir(), "index.rebuild.lock");
|
|
7544
|
-
}
|
|
7545
7543
|
function getMaintenanceBarrierPath() {
|
|
7546
7544
|
return path3.join(getDataDir(), "maintenance.barrier.lock");
|
|
7547
7545
|
}
|
|
@@ -8173,14 +8171,6 @@ var init_common = __esm(() => {
|
|
|
8173
8171
|
});
|
|
8174
8172
|
|
|
8175
8173
|
// src/core/recognition-util.ts
|
|
8176
|
-
function canonicalizeWorkflowName(name) {
|
|
8177
|
-
const lower = name.toLowerCase();
|
|
8178
|
-
for (const ext of WORKFLOW_EXTENSIONS) {
|
|
8179
|
-
if (lower.endsWith(ext))
|
|
8180
|
-
return name.slice(0, -ext.length);
|
|
8181
|
-
}
|
|
8182
|
-
return name;
|
|
8183
|
-
}
|
|
8184
8174
|
function isKnownType(type) {
|
|
8185
8175
|
return KNOWN_TYPES.includes(type);
|
|
8186
8176
|
}
|
|
@@ -8276,6 +8266,9 @@ function assetPathCandidatesForName(assetType, typeRoot, name) {
|
|
|
8276
8266
|
const namedForm = path7.join(typeRoot, base, "default.env");
|
|
8277
8267
|
return [...new Set([primary, dotForm, namedForm])];
|
|
8278
8268
|
}
|
|
8269
|
+
function assetPathCandidatesAreOrderedByPreference(assetType) {
|
|
8270
|
+
return assetType === "memory";
|
|
8271
|
+
}
|
|
8279
8272
|
var workflowSpec, markdownSpec, scriptSpec, BUILTIN_PLACEMENT_SPECS, PLACEMENT_SPECS;
|
|
8280
8273
|
var init_asset_placement = __esm(() => {
|
|
8281
8274
|
init_common();
|
|
@@ -11389,7 +11382,8 @@ function applyPostContributorFields(entry, file, canonicalName, dirPath) {
|
|
|
11389
11382
|
}
|
|
11390
11383
|
entry.tags = [...entry.tags ?? [], ...extractDirTagsFromName(canonicalName)];
|
|
11391
11384
|
entry.tags = normalizeTerms(entry.tags ?? []);
|
|
11392
|
-
|
|
11385
|
+
const aliasTagInput = entry.type === "memory" && canonicalName.toLowerCase().endsWith(".derived") ? entry.tags.filter((tag) => tag !== "derived") : entry.tags;
|
|
11386
|
+
entry.aliases = mergeAliases(entry.aliases, buildAliases(canonicalName, aliasTagInput));
|
|
11393
11387
|
entry.filename = path17.basename(file);
|
|
11394
11388
|
}
|
|
11395
11389
|
function buildMetadataSkipWarning(filePath, assetType, error) {
|
|
@@ -35706,7 +35700,7 @@ var require_libvips = __commonJS((exports, module) => {
|
|
|
35706
35700
|
SPDX-License-Identifier: Apache-2.0
|
|
35707
35701
|
*/
|
|
35708
35702
|
var { spawnSync: spawnSync6 } = __require("child_process");
|
|
35709
|
-
var { createHash:
|
|
35703
|
+
var { createHash: createHash10 } = __require("crypto");
|
|
35710
35704
|
var semverCoerce = require_coerce2();
|
|
35711
35705
|
var semverGreaterThanOrEqualTo = require_gte2();
|
|
35712
35706
|
var semverSatisfies = require_satisfies2();
|
|
@@ -35794,7 +35788,7 @@ var require_libvips = __commonJS((exports, module) => {
|
|
|
35794
35788
|
}
|
|
35795
35789
|
return false;
|
|
35796
35790
|
};
|
|
35797
|
-
var sha512 = (s) =>
|
|
35791
|
+
var sha512 = (s) => createHash10("sha512").update(s).digest("hex");
|
|
35798
35792
|
var yarnLocator = () => {
|
|
35799
35793
|
try {
|
|
35800
35794
|
const identHash = sha512(`imgsharp-libvips-${buildPlatformArch()}`);
|
|
@@ -43246,8 +43240,8 @@ function cos_sim(arr1, arr2) {
|
|
|
43246
43240
|
const dotProduct = dot(arr1, arr2);
|
|
43247
43241
|
const magnitudeA = magnitude(arr1);
|
|
43248
43242
|
const magnitudeB = magnitude(arr2);
|
|
43249
|
-
const
|
|
43250
|
-
return
|
|
43243
|
+
const cosineSimilarity = dotProduct / (magnitudeA * magnitudeB);
|
|
43244
|
+
return cosineSimilarity;
|
|
43251
43245
|
}
|
|
43252
43246
|
function magnitude(arr) {
|
|
43253
43247
|
return Math.sqrt(arr.reduce((acc, val) => acc + val * val, 0));
|
|
@@ -68186,7 +68180,7 @@ ${this.boa_token}${this.audio_token.repeat(this._compute_audio_num_tokens(audio_
|
|
|
68186
68180
|
});
|
|
68187
68181
|
|
|
68188
68182
|
// src/indexer/walk/file-context.ts
|
|
68189
|
-
import
|
|
68183
|
+
import fs49 from "fs";
|
|
68190
68184
|
import path61 from "path";
|
|
68191
68185
|
function buildFileContext(stashRoot, absPath) {
|
|
68192
68186
|
const relPath = toPosix(path61.relative(stashRoot, absPath));
|
|
@@ -68211,7 +68205,7 @@ function buildFileContext(stashRoot, absPath) {
|
|
|
68211
68205
|
stashRoot,
|
|
68212
68206
|
content() {
|
|
68213
68207
|
if (cachedContent === undefined) {
|
|
68214
|
-
cachedContent =
|
|
68208
|
+
cachedContent = fs49.readFileSync(absPath, "utf8");
|
|
68215
68209
|
}
|
|
68216
68210
|
return cachedContent;
|
|
68217
68211
|
},
|
|
@@ -68226,7 +68220,7 @@ function buildFileContext(stashRoot, absPath) {
|
|
|
68226
68220
|
},
|
|
68227
68221
|
stat() {
|
|
68228
68222
|
if (cachedStat === undefined) {
|
|
68229
|
-
cachedStat =
|
|
68223
|
+
cachedStat = fs49.statSync(absPath);
|
|
68230
68224
|
}
|
|
68231
68225
|
return cachedStat;
|
|
68232
68226
|
}
|
|
@@ -69024,6 +69018,7 @@ var PASSTHROUGH_COMMANDS = [
|
|
|
69024
69018
|
"improve-report",
|
|
69025
69019
|
"import",
|
|
69026
69020
|
"index",
|
|
69021
|
+
"index-status",
|
|
69027
69022
|
"info",
|
|
69028
69023
|
"lint",
|
|
69029
69024
|
"list",
|
|
@@ -69968,7 +69963,7 @@ Config saved to ${r.configPath}`;
|
|
|
69968
69963
|
}
|
|
69969
69964
|
function formatIndexPlain(r) {
|
|
69970
69965
|
const indexResult = r;
|
|
69971
|
-
let out = `Indexed ${indexResult.totalEntries ?? 0} entries from ${indexResult.
|
|
69966
|
+
let out = `Indexed ${indexResult.totalEntries ?? 0} entries from ${indexResult.sourcesScanned ?? 0} source${indexResult.sourcesScanned === 1 ? "" : "s"} (mode: ${indexResult.mode ?? "unknown"})`;
|
|
69972
69967
|
const warnings = indexResult.warnings;
|
|
69973
69968
|
if (Array.isArray(warnings) && warnings.length > 0) {
|
|
69974
69969
|
out += `
|
|
@@ -69977,13 +69972,6 @@ Warnings (${warnings.length}):`;
|
|
|
69977
69972
|
out += `
|
|
69978
69973
|
- ${String(message)}`;
|
|
69979
69974
|
}
|
|
69980
|
-
const notices = Array.isArray(indexResult.notices) ? indexResult.notices : [];
|
|
69981
|
-
for (const notice of notices) {
|
|
69982
|
-
const severity = notice.severity === "info" ? "info" : "warning";
|
|
69983
|
-
const field = typeof notice.field === "string" ? ` field=${notice.field}` : "";
|
|
69984
|
-
out += `
|
|
69985
|
-
notice[${severity}] ${notice.code} adapter=${notice.adapter}${field}` + (notice.message ? `: ${notice.message}` : "");
|
|
69986
|
-
}
|
|
69987
69975
|
const verification = indexResult.verification;
|
|
69988
69976
|
if (verification?.ok === false && verification.message) {
|
|
69989
69977
|
out += `
|
|
@@ -69992,10 +69980,26 @@ Verification: ${String(verification.message)}`;
|
|
|
69992
69980
|
const timing = indexResult.timing;
|
|
69993
69981
|
if (timing) {
|
|
69994
69982
|
out += `
|
|
69995
|
-
Timing: total ${timing.totalMs}ms` + `, preflight ${timing.preflightMs}ms` + `,
|
|
69983
|
+
Timing: total ${timing.totalMs}ms` + `, preflight ${timing.preflightMs}ms` + `, source cache ${timing.sourceCacheMs}ms` + `, reconcile ${timing.reconcileMs}ms` + `, embeddings ${timing.embedMs}ms` + `, finalize ${timing.finalizeMs}ms` + `, end-to-end ${timing.endToEndMs}ms`;
|
|
69996
69984
|
}
|
|
69997
69985
|
return out;
|
|
69998
69986
|
}
|
|
69987
|
+
function formatIndexStatusPlain(r) {
|
|
69988
|
+
const units = r.units ?? {};
|
|
69989
|
+
const lines = [
|
|
69990
|
+
`Index: ${String(r.indexPath ?? "unknown")}`,
|
|
69991
|
+
`Files: ${Number(r.files ?? 0)}`,
|
|
69992
|
+
`Entries: ${Number(r.entries ?? 0)}`,
|
|
69993
|
+
`Units: ${Number(units.total ?? 0)} total, ${Number(units.withVector ?? 0)} with a vector, ${Number(units.pending ?? 0)} pending`,
|
|
69994
|
+
`Active identity: ${typeof r.activeIdentity === "string" ? r.activeIdentity : "none yet"}`,
|
|
69995
|
+
`Last reconcile: ${typeof r.lastReconcileAt === "string" ? r.lastReconcileAt : "never"}`,
|
|
69996
|
+
`Built at: ${typeof r.builtAt === "string" ? r.builtAt : "never"}`
|
|
69997
|
+
];
|
|
69998
|
+
if (typeof r.unreadable === "string")
|
|
69999
|
+
lines.push(`Unreadable: ${r.unreadable}`);
|
|
70000
|
+
return lines.join(`
|
|
70001
|
+
`);
|
|
70002
|
+
}
|
|
69999
70003
|
function formatListPlain(r) {
|
|
70000
70004
|
const sources = Array.isArray(r.sources) ? r.sources : [];
|
|
70001
70005
|
if (sources.length === 0)
|
|
@@ -75248,10 +75252,6 @@ var EmbeddingConnectionConfigSchema = exports_external.object({
|
|
|
75248
75252
|
apiKey: symbolicOrWarnApiKey("embedding.apiKey").optional(),
|
|
75249
75253
|
dimension: positiveInt.max(4096).optional(),
|
|
75250
75254
|
localModel: exports_external.string().min(1).optional(),
|
|
75251
|
-
maxInputTokens: positiveInt.optional(),
|
|
75252
|
-
maxTokens: positiveInt.optional(),
|
|
75253
|
-
batchSize: positiveInt.optional(),
|
|
75254
|
-
contextLength: positiveInt.optional(),
|
|
75255
75255
|
ollamaOptions: EmbeddingOllamaOptionsSchema.optional(),
|
|
75256
75256
|
timeoutMs: positiveInt.optional(),
|
|
75257
75257
|
concurrency: positiveInt.max(16).optional()
|
|
@@ -75707,19 +75707,18 @@ var SearchGraphBoostSchema = exports_external.object({
|
|
|
75707
75707
|
confidenceMode: exports_external.enum(["blend"]).default("blend").optional(),
|
|
75708
75708
|
confidenceWeight: exports_external.number().finite().min(0).max(1).default(0.2).optional()
|
|
75709
75709
|
}).passthrough();
|
|
75710
|
-
var
|
|
75710
|
+
var SearchRerankConfigSchema = exports_external.object({
|
|
75711
75711
|
enabled: exports_external.boolean().optional(),
|
|
75712
75712
|
endpoint: httpUrl.optional(),
|
|
75713
75713
|
model: nonEmptyString.optional(),
|
|
75714
|
-
apiKey: symbolicOrWarnApiKey("search.
|
|
75714
|
+
apiKey: symbolicOrWarnApiKey("search.rerank.apiKey").optional(),
|
|
75715
75715
|
timeoutMs: positiveInt.optional(),
|
|
75716
75716
|
topN: positiveInt.max(50).optional()
|
|
75717
75717
|
}).passthrough();
|
|
75718
75718
|
var SearchConfigSchema = exports_external.object({
|
|
75719
|
-
minScore: nonNegativeNumber.optional(),
|
|
75720
75719
|
defaultExcludeTypes: exports_external.array(nonEmptyString).optional(),
|
|
75721
75720
|
graphBoost: SearchGraphBoostSchema.optional(),
|
|
75722
|
-
|
|
75721
|
+
rerank: SearchRerankConfigSchema.optional()
|
|
75723
75722
|
}).passthrough();
|
|
75724
75723
|
|
|
75725
75724
|
// src/core/config/schema/setup.ts
|
|
@@ -82895,7 +82894,7 @@ function foldRecognizedMetadata(rendererName, file) {
|
|
|
82895
82894
|
const fm = parseFrontmatter(file.content()).data;
|
|
82896
82895
|
applyFrontmatterDescriptionAndTags(fm, out);
|
|
82897
82896
|
const hints = new Set;
|
|
82898
|
-
const source = nonEmptyString2(fm.source);
|
|
82897
|
+
const source = fm.inferred === true ? undefined : nonEmptyString2(fm.source);
|
|
82899
82898
|
if (source)
|
|
82900
82899
|
hints.add(source);
|
|
82901
82900
|
const fmObservedAt = nonEmptyString2(fm.observed_at);
|
|
@@ -83344,9 +83343,23 @@ var akmAdapter = {
|
|
|
83344
83343
|
return [];
|
|
83345
83344
|
const canonical = assetPathCandidatesForName(type, path24.join(c.root, head), rest);
|
|
83346
83345
|
const loose = assetPathCandidatesForName(type, c.root, rest);
|
|
83347
|
-
|
|
83346
|
+
if (!assetPathCandidatesAreOrderedByPreference(type)) {
|
|
83347
|
+
return [...new Set([...canonical, ...loose])].map((candidatePath) => ({
|
|
83348
|
+
path: candidatePath,
|
|
83349
|
+
conceptId: posix
|
|
83350
|
+
}));
|
|
83351
|
+
}
|
|
83352
|
+
const priorityByPath = new Map;
|
|
83353
|
+
for (const list of [canonical, loose]) {
|
|
83354
|
+
list.forEach((candidatePath, rank) => {
|
|
83355
|
+
if (!priorityByPath.has(candidatePath))
|
|
83356
|
+
priorityByPath.set(candidatePath, rank);
|
|
83357
|
+
});
|
|
83358
|
+
}
|
|
83359
|
+
return [...priorityByPath.keys()].map((candidatePath) => ({
|
|
83348
83360
|
path: candidatePath,
|
|
83349
|
-
conceptId: posix
|
|
83361
|
+
conceptId: posix,
|
|
83362
|
+
priority: priorityByPath.get(candidatePath)
|
|
83350
83363
|
}));
|
|
83351
83364
|
},
|
|
83352
83365
|
placeNew(c, conceptId) {
|
|
@@ -93479,6 +93492,7 @@ import { AsyncLocalStorage as AsyncLocalStorage3 } from "async_hooks";
|
|
|
93479
93492
|
var attributionStorage = new AsyncLocalStorage3;
|
|
93480
93493
|
// src/llm/client.ts
|
|
93481
93494
|
var jsonSchemaUnsupportedConnections = new Set;
|
|
93495
|
+
var HEALTH_PROBE_TIMEOUT_MS = 3000;
|
|
93482
93496
|
|
|
93483
93497
|
// src/integrations/agent/model-map.ts
|
|
93484
93498
|
init_common();
|
|
@@ -93670,7 +93684,10 @@ var improveReportFormatters = [
|
|
|
93670
93684
|
];
|
|
93671
93685
|
|
|
93672
93686
|
// src/output/text/index.ts
|
|
93673
|
-
var indexFormatters = [
|
|
93687
|
+
var indexFormatters = [
|
|
93688
|
+
{ command: "index", handler: (r) => formatIndexPlain(r) },
|
|
93689
|
+
{ command: "index-status", handler: (r) => formatIndexStatusPlain(r) }
|
|
93690
|
+
];
|
|
93674
93691
|
|
|
93675
93692
|
// src/output/text/info.ts
|
|
93676
93693
|
var infoFormatters = [{ command: "info", handler: (r) => formatInfoPlain(r) }];
|
|
@@ -94940,6 +94957,9 @@ function runMigrations2(db, options) {
|
|
|
94940
94957
|
const initialMigration = STATE_MIGRATIONS[0];
|
|
94941
94958
|
if (!initialMigration)
|
|
94942
94959
|
throw new Error("State migration registry has no initial migration.");
|
|
94960
|
+
const finalMigration = STATE_MIGRATIONS[STATE_MIGRATIONS.length - 1];
|
|
94961
|
+
if (!finalMigration)
|
|
94962
|
+
throw new Error("State migration registry has no final migration.");
|
|
94943
94963
|
let existingUnversionedSnapshotPrepared = false;
|
|
94944
94964
|
const prepareExistingUnversionedState = (lockedDb) => {
|
|
94945
94965
|
if (existingUnversionedSnapshotPrepared)
|
|
@@ -94961,7 +94981,7 @@ function runMigrations2(db, options) {
|
|
|
94961
94981
|
existingUnversionedSnapshotPrepared = true;
|
|
94962
94982
|
};
|
|
94963
94983
|
runMigrations(db, STATE_MIGRATIONS, {
|
|
94964
|
-
lockInitialMigrationPrefixThrough: options?.existingUnversionedDatabase ? "002-task-history-per-run" : undefined,
|
|
94984
|
+
lockInitialMigrationPrefixThrough: options?.existingUnversionedDatabase ? "002-task-history-per-run" : options?.freshDatabase ? finalMigration.id : undefined,
|
|
94965
94985
|
beforeLedgerInitializationLocked(lockedDb) {
|
|
94966
94986
|
if (options?.freshDatabase)
|
|
94967
94987
|
return;
|
|
@@ -95222,10 +95242,13 @@ function openStateDatabase(dbPath, options) {
|
|
|
95222
95242
|
});
|
|
95223
95243
|
try {
|
|
95224
95244
|
preflight.exec("PRAGMA busy_timeout = 30000");
|
|
95225
|
-
const ledger =
|
|
95245
|
+
const { ledger, hasNoOtherTables } = preflight.transaction(() => ({
|
|
95246
|
+
ledger: assertMigrationLedger(preflight, STATE_MIGRATIONS),
|
|
95247
|
+
hasNoOtherTables: unversionedDatabaseHasNoTables(preflight)
|
|
95248
|
+
}))();
|
|
95226
95249
|
warnNewerStateLedger(ledger);
|
|
95227
95250
|
existingUnversionedDatabase = ledger.migrationIds.length === 0;
|
|
95228
|
-
if (existingUnversionedDatabase &&
|
|
95251
|
+
if (existingUnversionedDatabase && hasNoOtherTables) {
|
|
95229
95252
|
existingUnversionedDatabase = false;
|
|
95230
95253
|
treatUnversionedAsFresh = true;
|
|
95231
95254
|
}
|
|
@@ -95307,7 +95330,7 @@ function openStateDatabase(dbPath, options) {
|
|
|
95307
95330
|
if (freshReservation)
|
|
95308
95331
|
closeFileIdentity(freshReservation);
|
|
95309
95332
|
releaseActivity?.();
|
|
95310
|
-
|
|
95333
|
+
throwBeginFailure(error, "state");
|
|
95311
95334
|
}
|
|
95312
95335
|
}
|
|
95313
95336
|
function listPendingStateMigrations(dbPath = getStateDbPath()) {
|
|
@@ -95361,15 +95384,15 @@ function sleepSyncMs2(ms) {
|
|
|
95361
95384
|
return;
|
|
95362
95385
|
sleepSync(ms);
|
|
95363
95386
|
}
|
|
95364
|
-
function throwBeginFailure(err) {
|
|
95387
|
+
function throwBeginFailure(err, dbKind) {
|
|
95365
95388
|
if (isSqliteContentionError(err)) {
|
|
95366
|
-
const contended = new TransientError("akm's state database is busy (another akm process is writing it); retry shortly.", "STATE_DB_CONTENDED");
|
|
95389
|
+
const contended = dbKind === "index" ? new TransientError("akm's index database is busy (another akm process is writing it); retry shortly.", "INDEX_DB_CONTENDED") : new TransientError("akm's state database is busy (another akm process is writing it); retry shortly.", "STATE_DB_CONTENDED");
|
|
95367
95390
|
contended.cause = err;
|
|
95368
95391
|
throw contended;
|
|
95369
95392
|
}
|
|
95370
95393
|
throw err;
|
|
95371
95394
|
}
|
|
95372
|
-
function beginImmediateTransaction(db) {
|
|
95395
|
+
function beginImmediateTransaction(db, dbKind = "state") {
|
|
95373
95396
|
if (db.inTransaction) {
|
|
95374
95397
|
throw new Error("beginImmediateTransaction requires a connection with no active transaction");
|
|
95375
95398
|
}
|
|
@@ -95392,16 +95415,16 @@ function beginImmediateTransaction(db) {
|
|
|
95392
95415
|
sleepSyncMs2(2 ** (attempt - 1));
|
|
95393
95416
|
continue;
|
|
95394
95417
|
}
|
|
95395
|
-
throwBeginFailure(err);
|
|
95418
|
+
throwBeginFailure(err, dbKind);
|
|
95396
95419
|
}
|
|
95397
95420
|
}
|
|
95398
|
-
throwBeginFailure(lastBeginErr);
|
|
95421
|
+
throwBeginFailure(lastBeginErr, dbKind);
|
|
95399
95422
|
}
|
|
95400
|
-
function withImmediateTransaction(db, fn) {
|
|
95423
|
+
function withImmediateTransaction(db, fn, dbKind = "state") {
|
|
95401
95424
|
if (db.inTransaction) {
|
|
95402
95425
|
return fn();
|
|
95403
95426
|
}
|
|
95404
|
-
beginImmediateTransaction(db);
|
|
95427
|
+
beginImmediateTransaction(db, dbKind);
|
|
95405
95428
|
try {
|
|
95406
95429
|
const result = fn();
|
|
95407
95430
|
if (!db.inTransaction) {
|
|
@@ -95737,9 +95760,9 @@ function listTxnJournalsTolerant(predicate) {
|
|
|
95737
95760
|
|
|
95738
95761
|
// src/commands/proposal/repository.ts
|
|
95739
95762
|
import { createHash as createHash11, randomUUID as randomUUID6 } from "crypto";
|
|
95740
|
-
import
|
|
95763
|
+
import fs53 from "fs";
|
|
95741
95764
|
init_dist();
|
|
95742
|
-
import
|
|
95765
|
+
import path65 from "path";
|
|
95743
95766
|
|
|
95744
95767
|
// src/core/adapter/validate-context.ts
|
|
95745
95768
|
init_asset_placement();
|
|
@@ -95798,20 +95821,9 @@ init_warn();
|
|
|
95798
95821
|
init_write_provenance();
|
|
95799
95822
|
|
|
95800
95823
|
// src/indexer/index-written-assets.ts
|
|
95801
|
-
import
|
|
95802
|
-
import path64 from "path";
|
|
95824
|
+
import path63 from "path";
|
|
95803
95825
|
init_errors();
|
|
95804
95826
|
init_paths();
|
|
95805
|
-
|
|
95806
|
-
// src/core/run-lock.ts
|
|
95807
|
-
init_errors();
|
|
95808
|
-
function formatLockHolderPid(holder) {
|
|
95809
|
-
if (holder.pid === null)
|
|
95810
|
-
return "unknown";
|
|
95811
|
-
return holder.launcherPid !== null ? `${holder.pid} (launcher ${holder.launcherPid})` : String(holder.pid);
|
|
95812
|
-
}
|
|
95813
|
-
|
|
95814
|
-
// src/indexer/index-written-assets.ts
|
|
95815
95827
|
init_warn();
|
|
95816
95828
|
|
|
95817
95829
|
// src/storage/repositories/index-connection.ts
|
|
@@ -95820,7 +95832,7 @@ init_paths();
|
|
|
95820
95832
|
init_warn();
|
|
95821
95833
|
|
|
95822
95834
|
// src/storage/repositories/index-entry-schema.ts
|
|
95823
|
-
var CANONICAL_INDEX_DB_VERSION =
|
|
95835
|
+
var CANONICAL_INDEX_DB_VERSION = 24;
|
|
95824
95836
|
var CANONICAL_ENTRY_SCHEMA_FINGERPRINT = {
|
|
95825
95837
|
tableSql: "CREATE TABLE entries ( id INTEGER PRIMARY KEY AUTOINCREMENT, item_ref TEXT NOT NULL UNIQUE, bundle_id TEXT NOT NULL, component_id TEXT NOT NULL, concept_id TEXT NOT NULL, adapter_id TEXT NOT NULL, type TEXT NOT NULL, file_path TEXT NOT NULL, content_hash TEXT, document_json TEXT NOT NULL, search_text TEXT NOT NULL, derived_from TEXT )",
|
|
95826
95838
|
sqliteSequenceTable: true,
|
|
@@ -95972,9 +95984,7 @@ var CANONICAL_ENTRY_SCHEMA_FINGERPRINT = {
|
|
|
95972
95984
|
}
|
|
95973
95985
|
],
|
|
95974
95986
|
searchSurfaces: {
|
|
95975
|
-
|
|
95976
|
-
fragmentSourceSql: "CREATE TABLE entry_fragments ( entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE, safe_markdown TEXT NOT NULL )",
|
|
95977
|
-
fragmentsFtsSql: "CREATE VIRTUAL TABLE entry_fragments_fts USING fts5( entry_id UNINDEXED, fragment_id UNINDEXED, fragment_ordinal UNINDEXED, content, tokenize='porter unicode61' )"
|
|
95987
|
+
fragmentSourceSql: "CREATE TABLE entry_fragments ( entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE, safe_markdown TEXT NOT NULL )"
|
|
95978
95988
|
}
|
|
95979
95989
|
};
|
|
95980
95990
|
function sqlString(value) {
|
|
@@ -96024,9 +96034,7 @@ function readEntrySchemaFingerprint(db) {
|
|
|
96024
96034
|
columns,
|
|
96025
96035
|
indexes,
|
|
96026
96036
|
searchSurfaces: {
|
|
96027
|
-
|
|
96028
|
-
fragmentSourceSql: readNamedTableSql(db, "entry_fragments"),
|
|
96029
|
-
fragmentsFtsSql: readNamedTableSql(db, "entry_fragments_fts")
|
|
96037
|
+
fragmentSourceSql: readNamedTableSql(db, "entry_fragments")
|
|
96030
96038
|
}
|
|
96031
96039
|
};
|
|
96032
96040
|
}
|
|
@@ -96066,13 +96074,66 @@ function isCanonicalIndexGeneration(db) {
|
|
|
96066
96074
|
init_errors();
|
|
96067
96075
|
init_warn();
|
|
96068
96076
|
|
|
96069
|
-
// src/storage/repositories/
|
|
96070
|
-
|
|
96077
|
+
// src/storage/repositories/index-sql.ts
|
|
96078
|
+
var SQLITE_CHUNK_SIZE = 500;
|
|
96071
96079
|
|
|
96072
|
-
// src/storage/repositories/
|
|
96073
|
-
function
|
|
96074
|
-
|
|
96075
|
-
|
|
96080
|
+
// src/storage/repositories/files-repository.ts
|
|
96081
|
+
function rowToFileState(row) {
|
|
96082
|
+
return {
|
|
96083
|
+
path: row.path,
|
|
96084
|
+
bundleId: row.bundle_id,
|
|
96085
|
+
size: row.size,
|
|
96086
|
+
mtimeMs: row.mtime_ms,
|
|
96087
|
+
ctimeMs: row.ctime_ms,
|
|
96088
|
+
blobHash: row.blob_hash,
|
|
96089
|
+
adapterId: row.adapter_id
|
|
96090
|
+
};
|
|
96091
|
+
}
|
|
96092
|
+
function getFileState(db, path58) {
|
|
96093
|
+
const row = db.prepare("SELECT path, bundle_id, size, mtime_ms, ctime_ms, blob_hash, adapter_id FROM files WHERE path = ?").get(path58);
|
|
96094
|
+
return row ? rowToFileState(row) : undefined;
|
|
96095
|
+
}
|
|
96096
|
+
function upsertFileState(db, row) {
|
|
96097
|
+
db.prepare(`INSERT INTO files (path, bundle_id, size, mtime_ms, ctime_ms, blob_hash, adapter_id) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
96098
|
+
ON CONFLICT(path) DO UPDATE SET bundle_id = excluded.bundle_id, size = excluded.size,
|
|
96099
|
+
mtime_ms = excluded.mtime_ms, ctime_ms = excluded.ctime_ms, blob_hash = excluded.blob_hash,
|
|
96100
|
+
adapter_id = excluded.adapter_id`).run(row.path, row.bundleId, row.size, row.mtimeMs, row.ctimeMs, row.blobHash, row.adapterId);
|
|
96101
|
+
}
|
|
96102
|
+
function deleteFileStates(db, paths) {
|
|
96103
|
+
for (let offset = 0;offset < paths.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96104
|
+
const chunk = paths.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96105
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
96106
|
+
db.prepare(`DELETE FROM files WHERE path IN (${placeholders})`).run(...chunk);
|
|
96107
|
+
}
|
|
96108
|
+
}
|
|
96109
|
+
function insertNewUnitTexts(db, units) {
|
|
96110
|
+
if (units.length === 0)
|
|
96111
|
+
return { inserted: 0 };
|
|
96112
|
+
const insertText = db.prepare("INSERT OR IGNORE INTO unit_texts (unit_hash, kind, text) VALUES (?, ?, ?)");
|
|
96113
|
+
const insertFts = db.prepare("INSERT INTO units_fts (unit_hash, text) VALUES (?, ?)");
|
|
96114
|
+
let inserted = 0;
|
|
96115
|
+
for (const unit of units) {
|
|
96116
|
+
const result = insertText.run(unit.hash, unit.kind, unit.text);
|
|
96117
|
+
if (Number(result.changes) > 0) {
|
|
96118
|
+
insertFts.run(unit.hash, unit.text);
|
|
96119
|
+
inserted++;
|
|
96120
|
+
}
|
|
96121
|
+
}
|
|
96122
|
+
return { inserted };
|
|
96123
|
+
}
|
|
96124
|
+
function pruneOrphanUnitTextsForHashes(db, hashes) {
|
|
96125
|
+
const unique = [...new Set(hashes)];
|
|
96126
|
+
if (unique.length === 0)
|
|
96127
|
+
return { removed: 0 };
|
|
96128
|
+
let removed = 0;
|
|
96129
|
+
for (let offset = 0;offset < unique.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96130
|
+
const chunk = unique.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96131
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
96132
|
+
db.prepare(`DELETE FROM units_fts WHERE unit_hash IN (${placeholders}) AND NOT EXISTS ` + `(SELECT 1 FROM entry_units WHERE entry_units.unit_hash = units_fts.unit_hash)`).run(...chunk);
|
|
96133
|
+
const result = db.prepare(`DELETE FROM unit_texts WHERE unit_hash IN (${placeholders}) AND NOT EXISTS ` + `(SELECT 1 FROM entry_units WHERE entry_units.unit_hash = unit_texts.unit_hash)`).run(...chunk);
|
|
96134
|
+
removed += Number(result.changes);
|
|
96135
|
+
}
|
|
96136
|
+
return { removed };
|
|
96076
96137
|
}
|
|
96077
96138
|
|
|
96078
96139
|
// src/storage/repositories/index-meta-repository.ts
|
|
@@ -96083,112 +96144,16 @@ function getMeta(db, key) {
|
|
|
96083
96144
|
function setMeta(db, key, value) {
|
|
96084
96145
|
db.prepare("INSERT OR REPLACE INTO index_meta (key, value) VALUES (?, ?)").run(key, value);
|
|
96085
96146
|
}
|
|
96086
|
-
function deleteMeta(db, key) {
|
|
96087
|
-
db.prepare("DELETE FROM index_meta WHERE key = ?").run(key);
|
|
96088
|
-
}
|
|
96089
|
-
|
|
96090
|
-
// src/storage/repositories/index-sql.ts
|
|
96091
|
-
var SQLITE_CHUNK_SIZE = 500;
|
|
96092
|
-
|
|
96093
|
-
// src/storage/repositories/embedding-salvage-repository.ts
|
|
96094
|
-
function hashEmbeddableText(searchText) {
|
|
96095
|
-
return createHash9("sha256").update(searchText, "utf8").digest("hex");
|
|
96096
|
-
}
|
|
96097
|
-
function purgeEmbeddingSalvage(db) {
|
|
96098
|
-
db.exec("DELETE FROM embedding_salvage");
|
|
96099
|
-
}
|
|
96100
|
-
function relabelEmbeddingSalvageFingerprint(db, fromFingerprint, toFingerprint) {
|
|
96101
|
-
db.prepare("UPDATE embedding_salvage SET fingerprint = ? WHERE fingerprint = ?").run(toFingerprint, fromFingerprint);
|
|
96102
|
-
}
|
|
96103
|
-
function reuseSalvagedEmbeddings(db, entries, fingerprint, writeReused) {
|
|
96104
|
-
if (entries.length === 0)
|
|
96105
|
-
return { reusedCount: 0, remaining: [] };
|
|
96106
|
-
const anySalvageForFingerprint = db.prepare("SELECT 1 FROM embedding_salvage WHERE fingerprint = ? LIMIT 1").get(fingerprint);
|
|
96107
|
-
if (!anySalvageForFingerprint)
|
|
96108
|
-
return { reusedCount: 0, remaining: [...entries] };
|
|
96109
|
-
const hashes = entries.map((entry) => hashEmbeddableText(entry.searchText));
|
|
96110
|
-
const salvageByHash = new Map;
|
|
96111
|
-
const uniqueHashes = [...new Set(hashes)];
|
|
96112
|
-
for (let offset = 0;offset < uniqueHashes.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96113
|
-
const chunk = uniqueHashes.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96114
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
96115
|
-
const rows = db.prepare(`SELECT content_hash AS contentHash, embedding FROM embedding_salvage WHERE fingerprint = ? AND content_hash IN (${placeholders})`).all(fingerprint, ...chunk);
|
|
96116
|
-
for (const row of rows)
|
|
96117
|
-
salvageByHash.set(row.contentHash, row.embedding);
|
|
96118
|
-
}
|
|
96119
|
-
if (salvageByHash.size === 0)
|
|
96120
|
-
return { reusedCount: 0, remaining: [...entries] };
|
|
96121
|
-
let reusedCount = 0;
|
|
96122
|
-
const remaining = [];
|
|
96123
|
-
for (let offset = 0;offset < entries.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96124
|
-
const end = Math.min(offset + SQLITE_CHUNK_SIZE, entries.length);
|
|
96125
|
-
const chunkMatches = [];
|
|
96126
|
-
for (let i = offset;i < end; i++) {
|
|
96127
|
-
const entry = entries[i];
|
|
96128
|
-
const blob = salvageByHash.get(hashes[i]);
|
|
96129
|
-
if (blob)
|
|
96130
|
-
chunkMatches.push({ entry, blob });
|
|
96131
|
-
else
|
|
96132
|
-
remaining.push(entry);
|
|
96133
|
-
}
|
|
96134
|
-
if (chunkMatches.length === 0)
|
|
96135
|
-
continue;
|
|
96136
|
-
db.transaction(() => {
|
|
96137
|
-
for (const { entry, blob } of chunkMatches) {
|
|
96138
|
-
if (writeReused(entry, blobToEmbedding(blob)))
|
|
96139
|
-
reusedCount++;
|
|
96140
|
-
else
|
|
96141
|
-
remaining.push(entry);
|
|
96142
|
-
}
|
|
96143
|
-
})();
|
|
96144
|
-
}
|
|
96145
|
-
return { reusedCount, remaining };
|
|
96146
|
-
}
|
|
96147
96147
|
|
|
96148
96148
|
// src/storage/repositories/index-vec-repository.ts
|
|
96149
96149
|
import { createRequire as createRequire3 } from "module";
|
|
96150
|
-
|
|
96151
|
-
// src/core/best-effort.ts
|
|
96152
|
-
init_warn();
|
|
96153
|
-
function bestEffort(fn, context) {
|
|
96154
|
-
try {
|
|
96155
|
-
return fn();
|
|
96156
|
-
} catch (err) {
|
|
96157
|
-
if (isVerbose()) {
|
|
96158
|
-
warnVerbose(`[akm:best-effort] ${context ? `${context}: ` : ""}swallowed error`, err);
|
|
96159
|
-
}
|
|
96160
|
-
return;
|
|
96161
|
-
}
|
|
96162
|
-
}
|
|
96163
|
-
|
|
96164
|
-
// src/storage/repositories/index-vec-repository.ts
|
|
96165
|
-
init_warn();
|
|
96166
|
-
|
|
96167
|
-
// src/llm/embedders/types.ts
|
|
96168
|
-
init_warn();
|
|
96169
|
-
function cosineSimilarity(a, b) {
|
|
96170
|
-
if (a.length !== b.length) {
|
|
96171
|
-
warn("cosineSimilarity: vector dimension mismatch (%d vs %d) \u2014 re-index recommended", a.length, b.length);
|
|
96172
|
-
return 0;
|
|
96173
|
-
}
|
|
96174
|
-
const len = a.length;
|
|
96175
|
-
if (len === 0)
|
|
96176
|
-
return 0;
|
|
96177
|
-
let dot = 0;
|
|
96178
|
-
let magA = 0;
|
|
96179
|
-
let magB = 0;
|
|
96180
|
-
for (let i = 0;i < len; i++) {
|
|
96181
|
-
dot += a[i] * b[i];
|
|
96182
|
-
magA += a[i] * a[i];
|
|
96183
|
-
magB += b[i] * b[i];
|
|
96184
|
-
}
|
|
96185
|
-
const denom = Math.sqrt(magA) * Math.sqrt(magB);
|
|
96186
|
-
return denom === 0 ? 0 : dot / denom;
|
|
96187
|
-
}
|
|
96188
|
-
|
|
96189
|
-
// src/storage/repositories/index-vec-repository.ts
|
|
96190
96150
|
var vecStatus = new WeakMap;
|
|
96151
|
+
var forceVecUnavailableForTests = false;
|
|
96191
96152
|
function loadVecExtension(db) {
|
|
96153
|
+
if (forceVecUnavailableForTests) {
|
|
96154
|
+
vecStatus.set(db, false);
|
|
96155
|
+
return;
|
|
96156
|
+
}
|
|
96192
96157
|
try {
|
|
96193
96158
|
const esmRequire = createRequire3(import.meta.url);
|
|
96194
96159
|
const sqliteVec = esmRequire("sqlite-vec");
|
|
@@ -96201,135 +96166,135 @@ function loadVecExtension(db) {
|
|
|
96201
96166
|
function isVecAvailable(db) {
|
|
96202
96167
|
return vecStatus.get(db) ?? false;
|
|
96203
96168
|
}
|
|
96204
|
-
|
|
96205
|
-
|
|
96206
|
-
|
|
96169
|
+
|
|
96170
|
+
// src/storage/repositories/units-repository.ts
|
|
96171
|
+
init_errors();
|
|
96172
|
+
function createUnitsVecTable(db, dim) {
|
|
96173
|
+
db.exec(`
|
|
96174
|
+
CREATE VIRTUAL TABLE units_vec USING vec0(
|
|
96175
|
+
unit_id INTEGER PRIMARY KEY,
|
|
96176
|
+
embedding FLOAT[${dim}],
|
|
96177
|
+
+unit_hash TEXT,
|
|
96178
|
+
+identity TEXT
|
|
96179
|
+
);
|
|
96180
|
+
`);
|
|
96207
96181
|
}
|
|
96208
|
-
function
|
|
96209
|
-
|
|
96210
|
-
return false;
|
|
96211
|
-
return hasVecTable(db);
|
|
96182
|
+
function tableExists(db, name) {
|
|
96183
|
+
return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) != null;
|
|
96212
96184
|
}
|
|
96213
|
-
function
|
|
96214
|
-
|
|
96215
|
-
|
|
96216
|
-
|
|
96217
|
-
const missingVecRows = db.prepare(`
|
|
96218
|
-
SELECT id FROM embeddings
|
|
96219
|
-
EXCEPT
|
|
96220
|
-
SELECT id FROM entries_vec
|
|
96221
|
-
LIMIT 1
|
|
96222
|
-
`).all();
|
|
96223
|
-
if (missingVecRows.length > 0)
|
|
96224
|
-
return false;
|
|
96225
|
-
const orphanVecRows = db.prepare(`
|
|
96226
|
-
SELECT id FROM entries_vec
|
|
96227
|
-
EXCEPT
|
|
96228
|
-
SELECT id FROM embeddings
|
|
96229
|
-
LIMIT 1
|
|
96230
|
-
`).all();
|
|
96231
|
-
return orphanVecRows.length === 0;
|
|
96232
|
-
} catch {
|
|
96233
|
-
return false;
|
|
96234
|
-
}
|
|
96185
|
+
function unitsVecDimension(db) {
|
|
96186
|
+
const row = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'units_vec'").get();
|
|
96187
|
+
const match = row?.sql.match(/FLOAT\[(\d+)\]/);
|
|
96188
|
+
return match?.[1] === undefined ? undefined : Number(match[1]);
|
|
96235
96189
|
}
|
|
96236
|
-
|
|
96237
|
-
|
|
96238
|
-
|
|
96239
|
-
|
|
96240
|
-
|
|
96241
|
-
|
|
96242
|
-
|
|
96243
|
-
|
|
96244
|
-
|
|
96245
|
-
|
|
96246
|
-
|
|
96247
|
-
|
|
96248
|
-
|
|
96249
|
-
|
|
96250
|
-
|
|
96251
|
-
|
|
96252
|
-
|
|
96253
|
-
db.prepare("DELETE FROM entries_vec WHERE id = ?").run(id);
|
|
96254
|
-
}
|
|
96255
|
-
var vecInitWarnedDbs = new WeakSet;
|
|
96256
|
-
function purgeEmbeddings(db, opts) {
|
|
96257
|
-
bestEffort(() => db.exec("DELETE FROM embeddings"), "purge embeddings");
|
|
96258
|
-
if (isVecAvailable(db)) {
|
|
96259
|
-
bestEffort(() => db.exec(opts?.dropVecTable ? "DROP TABLE IF EXISTS entries_vec" : "DELETE FROM entries_vec"), "purge entries_vec");
|
|
96260
|
-
}
|
|
96261
|
-
setMeta(db, "hasEmbeddings", "0");
|
|
96262
|
-
}
|
|
96263
|
-
function upsertEmbedding(db, entryId, embedding) {
|
|
96264
|
-
const exists = db.prepare("SELECT 1 FROM entries WHERE id = ?").get(entryId);
|
|
96265
|
-
if (!exists)
|
|
96266
|
-
return { stored: false, vec: "unavailable" };
|
|
96267
|
-
const buf = float32Buffer(embedding);
|
|
96268
|
-
db.prepare("INSERT OR REPLACE INTO embeddings (id, embedding) VALUES (?, ?)").run(entryId, buf);
|
|
96190
|
+
function ensureUnitTables(db, dim) {
|
|
96191
|
+
db.exec(`
|
|
96192
|
+
CREATE TABLE IF NOT EXISTS units (
|
|
96193
|
+
unit_id INTEGER PRIMARY KEY,
|
|
96194
|
+
unit_hash TEXT NOT NULL,
|
|
96195
|
+
identity TEXT NOT NULL,
|
|
96196
|
+
UNIQUE (unit_hash, identity)
|
|
96197
|
+
);
|
|
96198
|
+
CREATE TABLE IF NOT EXISTS entry_units (
|
|
96199
|
+
entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE,
|
|
96200
|
+
ordinal INTEGER NOT NULL,
|
|
96201
|
+
fragment_id TEXT,
|
|
96202
|
+
unit_hash TEXT NOT NULL,
|
|
96203
|
+
PRIMARY KEY (entry_id, ordinal)
|
|
96204
|
+
);
|
|
96205
|
+
CREATE INDEX IF NOT EXISTS entry_units_hash ON entry_units(unit_hash);
|
|
96206
|
+
`);
|
|
96269
96207
|
if (!isVecAvailable(db))
|
|
96270
|
-
return
|
|
96271
|
-
|
|
96272
|
-
|
|
96273
|
-
|
|
96274
|
-
|
|
96275
|
-
|
|
96276
|
-
|
|
96277
|
-
|
|
96278
|
-
|
|
96208
|
+
return;
|
|
96209
|
+
if (tableExists(db, "units_vec"))
|
|
96210
|
+
return;
|
|
96211
|
+
createUnitsVecTable(db, dim);
|
|
96212
|
+
}
|
|
96213
|
+
function float32Buffer(vector) {
|
|
96214
|
+
return Buffer.from(new Float32Array(vector).buffer);
|
|
96215
|
+
}
|
|
96216
|
+
function upsertUnitVectors(db, rows) {
|
|
96217
|
+
if (rows.length === 0 || !isVecAvailable(db))
|
|
96218
|
+
return { inserted: 0, failed: 0 };
|
|
96219
|
+
const insertUnit = db.prepare("INSERT INTO units (unit_hash, identity) VALUES (?, ?) ON CONFLICT(unit_hash, identity) DO NOTHING");
|
|
96220
|
+
const selectUnitId = db.prepare("SELECT unit_id FROM units WHERE unit_hash = ? AND identity = ?");
|
|
96221
|
+
const deleteVec = db.prepare("DELETE FROM units_vec WHERE unit_id = ?");
|
|
96222
|
+
const insertVec = db.prepare("INSERT INTO units_vec (unit_id, embedding, unit_hash, identity) VALUES (?, ?, ?, ?)");
|
|
96223
|
+
const deleteUnit = db.prepare("DELETE FROM units WHERE unit_id = ?");
|
|
96224
|
+
const writeOne = (row) => withImmediateTransaction(db, () => {
|
|
96225
|
+
const result = insertUnit.run(row.hash, row.identity);
|
|
96226
|
+
const freshlyInserted = Number(result.changes) > 0;
|
|
96227
|
+
const unitRow = selectUnitId.get(row.hash, row.identity);
|
|
96228
|
+
if (!unitRow)
|
|
96229
|
+
return false;
|
|
96230
|
+
deleteVec.run(unitRow.unit_id);
|
|
96231
|
+
try {
|
|
96232
|
+
insertVec.run(unitRow.unit_id, float32Buffer(row.vector), row.hash, row.identity);
|
|
96233
|
+
} catch (err) {
|
|
96234
|
+
deleteUnit.run(unitRow.unit_id);
|
|
96235
|
+
throw err;
|
|
96236
|
+
}
|
|
96237
|
+
return freshlyInserted;
|
|
96238
|
+
}, "index");
|
|
96239
|
+
let inserted = 0;
|
|
96240
|
+
let failed = 0;
|
|
96241
|
+
for (const row of rows) {
|
|
96242
|
+
try {
|
|
96243
|
+
if (writeOne(row))
|
|
96244
|
+
inserted++;
|
|
96245
|
+
} catch {
|
|
96246
|
+
failed++;
|
|
96247
|
+
}
|
|
96279
96248
|
}
|
|
96249
|
+
return { inserted, failed };
|
|
96280
96250
|
}
|
|
96281
|
-
function
|
|
96282
|
-
const
|
|
96283
|
-
|
|
96284
|
-
|
|
96285
|
-
|
|
96286
|
-
|
|
96287
|
-
|
|
96288
|
-
return null;
|
|
96289
|
-
}
|
|
96290
|
-
const aligned = new ArrayBuffer(buf.byteLength);
|
|
96291
|
-
new Uint8Array(aligned).set(buf);
|
|
96292
|
-
const f32 = new Float32Array(aligned);
|
|
96293
|
-
return Array.from(f32);
|
|
96294
|
-
}
|
|
96295
|
-
function getAllEntriesForEmbedding(db, entryIds) {
|
|
96296
|
-
const select = `
|
|
96297
|
-
SELECT e.id, e.search_text AS searchText, e.item_ref AS itemRef, e.file_path AS filePath FROM entries e
|
|
96298
|
-
`;
|
|
96299
|
-
const missing = "NOT EXISTS (SELECT 1 FROM embeddings b WHERE b.id = e.id)";
|
|
96300
|
-
if (entryIds === undefined) {
|
|
96301
|
-
return db.prepare(`${select} WHERE ${missing} ORDER BY e.id`).all();
|
|
96302
|
-
}
|
|
96303
|
-
const targets = [...new Set(entryIds)].sort((left, right) => left - right);
|
|
96304
|
-
const rows = [];
|
|
96305
|
-
for (let offset = 0;offset < targets.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96306
|
-
const chunk = targets.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96307
|
-
if (chunk.length === 0)
|
|
96308
|
-
continue;
|
|
96251
|
+
function listMissingHashes(db, hashes, identity3) {
|
|
96252
|
+
const unique = [...new Set(hashes)];
|
|
96253
|
+
if (unique.length === 0)
|
|
96254
|
+
return [];
|
|
96255
|
+
const present = new Set;
|
|
96256
|
+
for (let offset = 0;offset < unique.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96257
|
+
const chunk = unique.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96309
96258
|
const placeholders = chunk.map(() => "?").join(",");
|
|
96310
|
-
rows
|
|
96259
|
+
const rows = db.prepare(`SELECT unit_hash FROM units WHERE identity = ? AND unit_hash IN (${placeholders})`).all(identity3, ...chunk);
|
|
96260
|
+
for (const row of rows)
|
|
96261
|
+
present.add(row.unit_hash);
|
|
96311
96262
|
}
|
|
96312
|
-
return
|
|
96263
|
+
return unique.filter((hash3) => !present.has(hash3));
|
|
96313
96264
|
}
|
|
96314
|
-
function
|
|
96315
|
-
|
|
96316
|
-
|
|
96265
|
+
function dropOtherIdentities(db, keep, dim) {
|
|
96266
|
+
if (!isVecAvailable(db))
|
|
96267
|
+
return { removed: 0 };
|
|
96268
|
+
ensureUnitTables(db, dim);
|
|
96269
|
+
const currentDim = unitsVecDimension(db);
|
|
96270
|
+
const widthChanged = currentDim !== undefined && currentDim !== dim;
|
|
96271
|
+
const staleCount = db.prepare("SELECT COUNT(*) AS n FROM units WHERE identity != ?").get(keep).n;
|
|
96272
|
+
if (staleCount === 0 && !widthChanged)
|
|
96273
|
+
return { removed: 0 };
|
|
96274
|
+
withImmediateTransaction(db, () => {
|
|
96275
|
+
if (widthChanged) {
|
|
96276
|
+
db.exec("DROP TABLE IF EXISTS units_vec");
|
|
96277
|
+
createUnitsVecTable(db, dim);
|
|
96278
|
+
} else {
|
|
96279
|
+
const staleIds = db.prepare("SELECT unit_id FROM units WHERE identity != ?").all(keep).map((row) => row.unit_id);
|
|
96280
|
+
for (let offset = 0;offset < staleIds.length; offset += SQLITE_CHUNK_SIZE) {
|
|
96281
|
+
const chunk = staleIds.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
96282
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
96283
|
+
db.prepare(`DELETE FROM units_vec WHERE unit_id IN (${placeholders})`).run(...chunk);
|
|
96284
|
+
}
|
|
96285
|
+
}
|
|
96286
|
+
db.prepare("DELETE FROM units WHERE identity != ?").run(keep);
|
|
96287
|
+
}, "index");
|
|
96288
|
+
return { removed: staleCount };
|
|
96317
96289
|
}
|
|
96318
|
-
function
|
|
96319
|
-
|
|
96320
|
-
|
|
96321
|
-
|
|
96322
|
-
|
|
96323
|
-
|
|
96324
|
-
|
|
96325
|
-
|
|
96326
|
-
const samples = [];
|
|
96327
|
-
for (const row of rows) {
|
|
96328
|
-
const vector = bufferToFloat32(row.embedding, Math.floor(row.embedding.byteLength / 4));
|
|
96329
|
-
if (vector)
|
|
96330
|
-
samples.push({ id: row.id, searchText: row.searchText, vector });
|
|
96331
|
-
}
|
|
96332
|
-
return samples;
|
|
96290
|
+
function replaceEntryUnits(db, entryId, units) {
|
|
96291
|
+
withImmediateTransaction(db, () => {
|
|
96292
|
+
db.prepare("DELETE FROM entry_units WHERE entry_id = ?").run(entryId);
|
|
96293
|
+
const insert = db.prepare("INSERT INTO entry_units (entry_id, ordinal, fragment_id, unit_hash) VALUES (?, ?, ?, ?)");
|
|
96294
|
+
for (const unit of units) {
|
|
96295
|
+
insert.run(entryId, unit.ordinal, unit.fragmentId, unit.hash);
|
|
96296
|
+
}
|
|
96297
|
+
}, "index");
|
|
96333
96298
|
}
|
|
96334
96299
|
|
|
96335
96300
|
// src/storage/repositories/index-connection.ts
|
|
@@ -96378,6 +96343,21 @@ function closeDatabase(db) {
|
|
|
96378
96343
|
import fs46 from "fs";
|
|
96379
96344
|
init_asset_ref();
|
|
96380
96345
|
init_resolve_ref();
|
|
96346
|
+
|
|
96347
|
+
// src/core/best-effort.ts
|
|
96348
|
+
init_warn();
|
|
96349
|
+
function bestEffort(fn, context) {
|
|
96350
|
+
try {
|
|
96351
|
+
return fn();
|
|
96352
|
+
} catch (err) {
|
|
96353
|
+
if (isVerbose()) {
|
|
96354
|
+
warnVerbose(`[akm:best-effort] ${context ? `${context}: ` : ""}swallowed error`, err);
|
|
96355
|
+
}
|
|
96356
|
+
return;
|
|
96357
|
+
}
|
|
96358
|
+
}
|
|
96359
|
+
|
|
96360
|
+
// src/storage/repositories/index-entries-repository.ts
|
|
96381
96361
|
init_warn();
|
|
96382
96362
|
init_metadata();
|
|
96383
96363
|
|
|
@@ -96445,11 +96425,11 @@ init_warn();
|
|
|
96445
96425
|
|
|
96446
96426
|
// src/core/asset/markdown-fragments.ts
|
|
96447
96427
|
init_markdown();
|
|
96448
|
-
import { createHash as
|
|
96428
|
+
import { createHash as createHash9 } from "crypto";
|
|
96449
96429
|
var MARKDOWN_FRAGMENT_MAX_CHARS = 1600;
|
|
96450
96430
|
var MARKDOWN_FRAGMENT_PREFIX = "akm-fragment-";
|
|
96451
96431
|
function hash3(text) {
|
|
96452
|
-
return
|
|
96432
|
+
return createHash9("sha256").update(text).digest("hex");
|
|
96453
96433
|
}
|
|
96454
96434
|
function uniqueSlugs(body) {
|
|
96455
96435
|
const out = new Map;
|
|
@@ -96563,48 +96543,30 @@ function splitMarkdownFragments(body, maxChars = MARKDOWN_FRAGMENT_MAX_CHARS) {
|
|
|
96563
96543
|
}
|
|
96564
96544
|
|
|
96565
96545
|
// src/storage/repositories/index-fts-repository.ts
|
|
96566
|
-
|
|
96567
|
-
|
|
96568
|
-
|
|
96569
|
-
var ftsMutationStatementsByDb = new WeakMap;
|
|
96570
|
-
function getFtsMutationStatements(db) {
|
|
96571
|
-
const existing = ftsMutationStatementsByDb.get(db);
|
|
96546
|
+
var fragmentSourceStatementsByDb = new WeakMap;
|
|
96547
|
+
function getFragmentSourceStatements(db) {
|
|
96548
|
+
const existing = fragmentSourceStatementsByDb.get(db);
|
|
96572
96549
|
if (existing)
|
|
96573
96550
|
return existing;
|
|
96574
96551
|
const statements = {
|
|
96575
|
-
deleteOne: db.prepare("DELETE FROM entries_fts WHERE entry_id = ?"),
|
|
96576
|
-
insert: db.prepare(INSERT_FTS_SQL),
|
|
96577
|
-
deleteFragments: db.prepare("DELETE FROM entry_fragments_fts WHERE entry_id = ?"),
|
|
96578
96552
|
upsertFragmentSource: db.prepare("INSERT INTO entry_fragments (entry_id, safe_markdown) VALUES (?, ?) ON CONFLICT(entry_id) DO UPDATE SET safe_markdown = excluded.safe_markdown"),
|
|
96579
|
-
deleteFragmentSource: db.prepare("DELETE FROM entry_fragments WHERE entry_id = ?")
|
|
96580
|
-
insertFragment: db.prepare(INSERT_FRAGMENT_SQL)
|
|
96553
|
+
deleteFragmentSource: db.prepare("DELETE FROM entry_fragments WHERE entry_id = ?")
|
|
96581
96554
|
};
|
|
96582
|
-
|
|
96555
|
+
fragmentSourceStatementsByDb.set(db, statements);
|
|
96583
96556
|
return statements;
|
|
96584
96557
|
}
|
|
96585
|
-
function
|
|
96586
|
-
|
|
96587
|
-
const statements = getFtsMutationStatements(db);
|
|
96588
|
-
statements.deleteOne.run(entryId);
|
|
96589
|
-
statements.insert.run(entryId, fields.name, fields.description, fields.tags, fields.hints, fields.content);
|
|
96590
|
-
if (fragmentContent === undefined) {
|
|
96558
|
+
function replaceFragmentSource(db, entryId, fragmentContent) {
|
|
96559
|
+
if (fragmentContent === undefined)
|
|
96591
96560
|
return;
|
|
96592
|
-
|
|
96593
|
-
statements.deleteFragments.run(entryId);
|
|
96561
|
+
const statements = getFragmentSourceStatements(db);
|
|
96594
96562
|
statements.deleteFragmentSource.run(entryId);
|
|
96595
|
-
if (
|
|
96596
|
-
|
|
96597
|
-
statements.upsertFragmentSource.run(entryId, fragmentContent);
|
|
96598
|
-
for (const fragment of splitMarkdownFragments(fragmentContent)) {
|
|
96599
|
-
statements.insertFragment.run(entryId, fragment.fragmentId, fragment.ordinal, fragment.text.toLowerCase());
|
|
96600
|
-
}
|
|
96563
|
+
if (fragmentContent)
|
|
96564
|
+
statements.upsertFragmentSource.run(entryId, fragmentContent);
|
|
96601
96565
|
}
|
|
96602
|
-
function
|
|
96566
|
+
function deleteFragmentSource(db, entryIds) {
|
|
96603
96567
|
for (let i = 0;i < entryIds.length; i += SQLITE_CHUNK_SIZE) {
|
|
96604
96568
|
const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
|
|
96605
96569
|
const placeholders = chunk.map(() => "?").join(",");
|
|
96606
|
-
db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
|
|
96607
|
-
db.prepare(`DELETE FROM entry_fragments_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
|
|
96608
96570
|
db.prepare(`DELETE FROM entry_fragments WHERE entry_id IN (${placeholders})`).run(...chunk);
|
|
96609
96571
|
}
|
|
96610
96572
|
}
|
|
@@ -96614,16 +96576,13 @@ function upsertEntry(db, filePath, entry, searchText, provenance, contentHash) {
|
|
|
96614
96576
|
const stmts = getUpsertStmts(db);
|
|
96615
96577
|
const derivedFrom = typeof entry.derivedFrom === "string" && entry.derivedFrom.trim() ? entry.derivedFrom.trim() : null;
|
|
96616
96578
|
const apply = () => {
|
|
96617
|
-
const previous = stmts.findByItemRef.get(provenance.itemRef);
|
|
96618
96579
|
const result = stmts.upsert.get(provenance.itemRef, provenance.bundleId, provenance.componentId, provenance.conceptId, provenance.adapterId, entry.type, filePath, contentHash ?? null, JSON.stringify(entry), searchText, derivedFrom);
|
|
96619
96580
|
if (!result)
|
|
96620
96581
|
throw new Error("upsertEntry: item_ref not found after upsert");
|
|
96621
|
-
|
|
96622
|
-
deleteEntryVectors(db, result.id);
|
|
96623
|
-
replaceFtsEntry(db, result.id, entry, hasMarkdownFragmentContent(entry) ? getMarkdownFragmentContent(entry) ?? null : undefined);
|
|
96582
|
+
replaceFragmentSource(db, result.id, hasMarkdownFragmentContent(entry) ? getMarkdownFragmentContent(entry) ?? null : undefined);
|
|
96624
96583
|
return result.id;
|
|
96625
96584
|
};
|
|
96626
|
-
return db.transaction(apply)();
|
|
96585
|
+
return db.inTransaction ? db.transaction(apply)() : withImmediateTransaction(db, apply, "index");
|
|
96627
96586
|
}
|
|
96628
96587
|
var upsertStmtsByDb = new WeakMap;
|
|
96629
96588
|
var UPSERT_SET_CLAUSE = `SET
|
|
@@ -96650,8 +96609,7 @@ function getUpsertStmts(db) {
|
|
|
96650
96609
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
96651
96610
|
ON CONFLICT(item_ref) DO UPDATE ${UPSERT_SET_CLAUSE}
|
|
96652
96611
|
RETURNING id
|
|
96653
|
-
`)
|
|
96654
|
-
findByItemRef: db.prepare("SELECT id, search_text FROM entries WHERE item_ref = ?")
|
|
96612
|
+
`)
|
|
96655
96613
|
};
|
|
96656
96614
|
upsertStmtsByDb.set(db, stmts);
|
|
96657
96615
|
return stmts;
|
|
@@ -96660,15 +96618,10 @@ function deleteRelatedRows(db, ids, options = {}) {
|
|
|
96660
96618
|
if (ids.length === 0)
|
|
96661
96619
|
return;
|
|
96662
96620
|
const numericIds = ids.map((r) => r.id);
|
|
96663
|
-
|
|
96664
|
-
deleteFtsEntries(db, numericIds);
|
|
96621
|
+
deleteFragmentSource(db, numericIds);
|
|
96665
96622
|
for (let i = 0;i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
|
|
96666
96623
|
const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
|
|
96667
96624
|
const placeholders = chunk.map(() => "?").join(",");
|
|
96668
|
-
bestEffort(() => db.prepare(`DELETE FROM embeddings WHERE id IN (${placeholders})`).run(...chunk), "delete embeddings for entries");
|
|
96669
|
-
if (vecAvail) {
|
|
96670
|
-
bestEffort(() => db.prepare(`DELETE FROM entries_vec WHERE id IN (${placeholders})`).run(...chunk), "delete entries_vec for entries");
|
|
96671
|
-
}
|
|
96672
96625
|
bestEffort(() => db.prepare(`DELETE FROM utility_scores WHERE entry_id IN (${placeholders})`).run(...chunk), "delete utility_scores for entries");
|
|
96673
96626
|
bestEffort(() => db.prepare(`DELETE FROM utility_scores_scoped WHERE entry_id IN (${placeholders})`).run(...chunk), "delete utility_scores_scoped for entries");
|
|
96674
96627
|
}
|
|
@@ -96711,7 +96664,7 @@ function deleteUsageEventsByEntryIds(entryIds) {
|
|
|
96711
96664
|
function deleteEntriesByIds(db, ids) {
|
|
96712
96665
|
if (ids.length === 0)
|
|
96713
96666
|
return;
|
|
96714
|
-
db
|
|
96667
|
+
withImmediateTransaction(db, () => {
|
|
96715
96668
|
const idObjs = ids.map((id) => ({ id }));
|
|
96716
96669
|
deleteRelatedRows(db, idObjs);
|
|
96717
96670
|
for (let i = 0;i < ids.length; i += SQLITE_CHUNK_SIZE) {
|
|
@@ -96719,17 +96672,14 @@ function deleteEntriesByIds(db, ids) {
|
|
|
96719
96672
|
const placeholders = chunk.map(() => "?").join(",");
|
|
96720
96673
|
db.prepare(`DELETE FROM entries WHERE id IN (${placeholders})`).run(...chunk);
|
|
96721
96674
|
}
|
|
96722
|
-
})
|
|
96675
|
+
}, "index");
|
|
96723
96676
|
}
|
|
96724
96677
|
function getEntryCount(db) {
|
|
96725
96678
|
const row = db.prepare("SELECT COUNT(*) AS cnt FROM entries").get();
|
|
96726
96679
|
return row.cnt;
|
|
96727
96680
|
}
|
|
96728
|
-
function getEmbeddableEntryCount(db) {
|
|
96729
|
-
return getEntryCount(db);
|
|
96730
|
-
}
|
|
96731
96681
|
|
|
96732
|
-
// src/indexer/
|
|
96682
|
+
// src/indexer/drain.ts
|
|
96733
96683
|
init_paths();
|
|
96734
96684
|
init_warn();
|
|
96735
96685
|
|
|
@@ -96970,19 +96920,6 @@ var DEFAULT_TOKEN_BUDGET = 6000;
|
|
|
96970
96920
|
function estimateTokenCount(text) {
|
|
96971
96921
|
return Math.round(text.length / 4);
|
|
96972
96922
|
}
|
|
96973
|
-
var DEFAULT_MAX_INPUT_TOKENS = 512;
|
|
96974
|
-
function capEmbeddingText(text, maxTokens) {
|
|
96975
|
-
if (estimateTokenCount(text) <= maxTokens)
|
|
96976
|
-
return { text, truncated: false };
|
|
96977
|
-
const charBudget = Math.max(0, maxTokens * 4);
|
|
96978
|
-
let cut = Math.min(charBudget, text.length);
|
|
96979
|
-
if (cut > 0 && cut < text.length) {
|
|
96980
|
-
const code = text.charCodeAt(cut);
|
|
96981
|
-
if (code >= 56320 && code <= 57343)
|
|
96982
|
-
cut -= 1;
|
|
96983
|
-
}
|
|
96984
|
-
return { text: text.slice(0, cut), truncated: true };
|
|
96985
|
-
}
|
|
96986
96923
|
var DEFAULT_EMBEDDING_TIMEOUT_MS = 120000;
|
|
96987
96924
|
function resolveEmbeddingTimeoutMs(config) {
|
|
96988
96925
|
return config.timeoutMs ?? DEFAULT_EMBEDDING_TIMEOUT_MS;
|
|
@@ -97021,7 +96958,7 @@ function resolveEmbeddingConcurrency(config) {
|
|
|
97021
96958
|
return config.concurrency;
|
|
97022
96959
|
return defaultConcurrencyForEndpoint(config.endpoint);
|
|
97023
96960
|
}
|
|
97024
|
-
function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
|
|
96961
|
+
function buildTokenBoundedBatches(texts, tokenBudget, maxCount, tokenCounts) {
|
|
97025
96962
|
const batches = [];
|
|
97026
96963
|
let current = [];
|
|
97027
96964
|
let currentTokens = 0;
|
|
@@ -97033,7 +96970,7 @@ function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
|
|
|
97033
96970
|
}
|
|
97034
96971
|
};
|
|
97035
96972
|
for (let i = 0;i < texts.length; i++) {
|
|
97036
|
-
const tokens = estimateTokenCount(texts[i]);
|
|
96973
|
+
const tokens = tokenCounts?.[i] ?? estimateTokenCount(texts[i]);
|
|
97037
96974
|
if (tokens > tokenBudget) {
|
|
97038
96975
|
flush();
|
|
97039
96976
|
batches.push({ indices: [i], oversized: true });
|
|
@@ -97049,7 +96986,7 @@ function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
|
|
|
97049
96986
|
return batches;
|
|
97050
96987
|
}
|
|
97051
96988
|
var ADAPTIVE_BUDGET_SHRINK_FACTOR = 0.75;
|
|
97052
|
-
var
|
|
96989
|
+
var ADAPTIVE_BUDGET_FLOOR_TOKENS = 1024;
|
|
97053
96990
|
|
|
97054
96991
|
class RemoteEmbedder {
|
|
97055
96992
|
config;
|
|
@@ -97076,6 +97013,9 @@ class RemoteEmbedder {
|
|
|
97076
97013
|
if (ollamaOpts) {
|
|
97077
97014
|
body.options = ollamaOpts;
|
|
97078
97015
|
}
|
|
97016
|
+
if (isOllamaNativeEmbedEndpoint(this.endpoint)) {
|
|
97017
|
+
body.truncate = false;
|
|
97018
|
+
}
|
|
97079
97019
|
const timeoutMs = resolveEmbeddingTimeoutMs(this.config);
|
|
97080
97020
|
const response = await fetchWithTimeout(normalizeEmbeddingEndpoint(this.endpoint), {
|
|
97081
97021
|
method: "POST",
|
|
@@ -97096,30 +97036,31 @@ class RemoteEmbedder {
|
|
|
97096
97036
|
}
|
|
97097
97037
|
return l2Normalize(json.data[0].embedding);
|
|
97098
97038
|
}
|
|
97099
|
-
async embedBatch(texts, signal, onSkip, onBatch) {
|
|
97039
|
+
async embedBatch(texts, signal, onSkip, onBatch, packing) {
|
|
97100
97040
|
if (texts.length === 0)
|
|
97101
97041
|
return [];
|
|
97102
97042
|
const results = new Array(texts.length).fill(undefined);
|
|
97103
97043
|
const headers = this.buildHeaders();
|
|
97104
|
-
const ollamaOpts = resolveOllamaOptions(this.config);
|
|
97105
|
-
|
|
97106
|
-
|
|
97107
|
-
const
|
|
97108
|
-
const
|
|
97044
|
+
const ollamaOpts = resolveOllamaOptions(this.config, packing?.ollamaNumCtx);
|
|
97045
|
+
const tokenCounts = texts.map((text) => Math.ceil(text.length / (packing?.charsPerToken ?? 4)));
|
|
97046
|
+
let effectiveTokenBudget = packing?.tokenBudget ?? DEFAULT_TOKEN_BUDGET;
|
|
97047
|
+
const maxCount = packing?.maxCount ?? DEFAULT_REMOTE_BATCH_SIZE;
|
|
97048
|
+
const windowIsKnown = packing?.windowIsKnown ?? false;
|
|
97049
|
+
const textBatches = buildTokenBoundedBatches(texts, effectiveTokenBudget, maxCount, tokenCounts);
|
|
97109
97050
|
const configuredTimeoutMs = resolveEmbeddingTimeoutMs(this.config);
|
|
97110
97051
|
let dispatchedBatchCount = 0;
|
|
97111
|
-
let budgetShrunk =
|
|
97052
|
+
let budgetShrunk = windowIsKnown;
|
|
97112
97053
|
const maybeShrinkBudget = (rejectedIndices, rejectedBatchIndex, rejectedRequestTokens) => {
|
|
97113
97054
|
if (budgetShrunk)
|
|
97114
97055
|
return;
|
|
97115
97056
|
budgetShrunk = true;
|
|
97116
|
-
|
|
97117
|
-
effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), floor2);
|
|
97057
|
+
effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), ADAPTIVE_BUDGET_FLOOR_TOKENS);
|
|
97118
97058
|
const notYetDispatched = textBatches.slice(dispatchedBatchCount);
|
|
97119
97059
|
const remainingIndices = notYetDispatched.flatMap((batch) => batch.indices);
|
|
97120
97060
|
if (remainingIndices.length > 0) {
|
|
97121
97061
|
const remainingTexts = remainingIndices.map((i) => texts[i]);
|
|
97122
|
-
const
|
|
97062
|
+
const remainingCounts = remainingIndices.map((i) => tokenCounts[i]);
|
|
97063
|
+
const replanned = buildTokenBoundedBatches(remainingTexts, effectiveTokenBudget, maxCount, remainingCounts).map((batch) => ({
|
|
97123
97064
|
indices: batch.indices.map((localIndex) => remainingIndices[localIndex]),
|
|
97124
97065
|
oversized: batch.oversized
|
|
97125
97066
|
}));
|
|
@@ -97169,7 +97110,7 @@ class RemoteEmbedder {
|
|
|
97169
97110
|
if (dispatchAbort.signal.aborted)
|
|
97170
97111
|
return;
|
|
97171
97112
|
const batch = indices.map((i) => texts[i]);
|
|
97172
|
-
const requestTokens =
|
|
97113
|
+
const requestTokens = indices.reduce((sum, i) => sum + tokenCounts[i], 0);
|
|
97173
97114
|
const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, effectiveTokenBudget);
|
|
97174
97115
|
const requestStart = Date.now();
|
|
97175
97116
|
let batchEmbeddings;
|
|
@@ -97258,7 +97199,7 @@ class RemoteEmbedder {
|
|
|
97258
97199
|
dispatchedBatchCount = batchIndex;
|
|
97259
97200
|
if (textBatch.oversized) {
|
|
97260
97201
|
const idx = textBatch.indices[0];
|
|
97261
|
-
const estTokens =
|
|
97202
|
+
const estTokens = tokenCounts[idx];
|
|
97262
97203
|
onSkip?.({
|
|
97263
97204
|
index: idx,
|
|
97264
97205
|
reason: "context-window-exceeded",
|
|
@@ -97304,6 +97245,9 @@ class RemoteEmbedder {
|
|
|
97304
97245
|
if (ollamaOpts) {
|
|
97305
97246
|
body.options = ollamaOpts;
|
|
97306
97247
|
}
|
|
97248
|
+
if (isOllamaNativeEmbedEndpoint(this.endpoint)) {
|
|
97249
|
+
body.truncate = false;
|
|
97250
|
+
}
|
|
97307
97251
|
const response = await fetchWithTimeout(normalizeEmbeddingEndpoint(this.endpoint), {
|
|
97308
97252
|
method: "POST",
|
|
97309
97253
|
headers,
|
|
@@ -97371,6 +97315,15 @@ function normalizeEmbeddingEndpoint(endpoint) {
|
|
|
97371
97315
|
parsed.pathname = normalizedPath ? `${normalizedPath}/embeddings` : "/embeddings";
|
|
97372
97316
|
return parsed.toString();
|
|
97373
97317
|
}
|
|
97318
|
+
function isOllamaNativeEmbedEndpoint(endpoint) {
|
|
97319
|
+
let parsed;
|
|
97320
|
+
try {
|
|
97321
|
+
parsed = new URL(normalizeEmbeddingEndpoint(endpoint));
|
|
97322
|
+
} catch {
|
|
97323
|
+
return false;
|
|
97324
|
+
}
|
|
97325
|
+
return parsed.pathname.replace(/\/+$/, "").endsWith("/embed");
|
|
97326
|
+
}
|
|
97374
97327
|
function embeddingEndpointPathHint(endpoint) {
|
|
97375
97328
|
const normalizedEndpoint = normalizeEmbeddingEndpoint(endpoint);
|
|
97376
97329
|
if (normalizedEndpoint !== endpoint) {
|
|
@@ -97378,12 +97331,12 @@ function embeddingEndpointPathHint(endpoint) {
|
|
|
97378
97331
|
}
|
|
97379
97332
|
return "";
|
|
97380
97333
|
}
|
|
97381
|
-
function resolveOllamaOptions(config) {
|
|
97334
|
+
function resolveOllamaOptions(config, ollamaNumCtx) {
|
|
97382
97335
|
if (config.ollamaOptions && Object.keys(config.ollamaOptions).length > 0) {
|
|
97383
97336
|
return config.ollamaOptions;
|
|
97384
97337
|
}
|
|
97385
|
-
if (
|
|
97386
|
-
return { num_ctx:
|
|
97338
|
+
if (ollamaNumCtx) {
|
|
97339
|
+
return { num_ctx: ollamaNumCtx };
|
|
97387
97340
|
}
|
|
97388
97341
|
return;
|
|
97389
97342
|
}
|
|
@@ -97399,6 +97352,8 @@ function describeEmbeddingCredential(apiKey) {
|
|
|
97399
97352
|
return `${apiKey} (env)`;
|
|
97400
97353
|
return "literal apiKey";
|
|
97401
97354
|
}
|
|
97355
|
+
// src/llm/embedders/types.ts
|
|
97356
|
+
init_warn();
|
|
97402
97357
|
|
|
97403
97358
|
// src/llm/embedder.ts
|
|
97404
97359
|
var embedderOverrides;
|
|
@@ -97409,9 +97364,9 @@ function getLocalEmbedder() {
|
|
|
97409
97364
|
}
|
|
97410
97365
|
return _localEmbedder;
|
|
97411
97366
|
}
|
|
97412
|
-
async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch) {
|
|
97367
|
+
async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch, packing) {
|
|
97413
97368
|
if (embedderOverrides?.embedBatch) {
|
|
97414
|
-
return embedderOverrides.embedBatch(texts, embeddingConfig, signal, onSkip, onBatch);
|
|
97369
|
+
return embedderOverrides.embedBatch(texts, embeddingConfig, signal, onSkip, onBatch, packing);
|
|
97415
97370
|
}
|
|
97416
97371
|
if (texts.length === 0)
|
|
97417
97372
|
return [];
|
|
@@ -97421,7 +97376,7 @@ async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch) {
|
|
|
97421
97376
|
return embeddings;
|
|
97422
97377
|
}
|
|
97423
97378
|
if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
|
|
97424
|
-
return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal, onSkip, onBatch);
|
|
97379
|
+
return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal, onSkip, onBatch, packing);
|
|
97425
97380
|
}
|
|
97426
97381
|
const localModel = embeddingConfig?.localModel;
|
|
97427
97382
|
if (!localModel) {
|
|
@@ -97439,70 +97394,185 @@ async function embedBatch(texts, embeddingConfig, signal, onSkip, onBatch) {
|
|
|
97439
97394
|
return results;
|
|
97440
97395
|
}
|
|
97441
97396
|
|
|
97442
|
-
// src/
|
|
97443
|
-
|
|
97444
|
-
|
|
97445
|
-
|
|
97446
|
-
|
|
97447
|
-
|
|
97448
|
-
|
|
97449
|
-
|
|
97397
|
+
// src/llm/embedders/provider-limits.ts
|
|
97398
|
+
var DEFAULT_WINDOW_TOKENS = 8192;
|
|
97399
|
+
var CHARS_PER_TOKEN_TAIL = 2.6;
|
|
97400
|
+
var UNIT_HEADER_MARGIN_TOKENS = 64;
|
|
97401
|
+
var CALIBRATION_PERCENTILE = 0.01;
|
|
97402
|
+
var TOKENIZE_PRESENCE_PROBE_TEXT = "ping";
|
|
97403
|
+
var CALIBRATION_SHAPES = [
|
|
97404
|
+
"This is a short sentence describing typical prose content used to calibrate the tokenizer.",
|
|
97405
|
+
"```ts\nfunction add(a: number, b: number): number {\n return a + b;\n}\n```",
|
|
97406
|
+
`| Column A | Column B | Column C |
|
|
97407
|
+
| --- | --- | --- |
|
|
97408
|
+
| 1 | 2 | 3 |
|
|
97409
|
+
| 4 | 5 | 6 |`,
|
|
97410
|
+
`- first item in a list
|
|
97411
|
+
- second item in a list
|
|
97412
|
+
- third item, a little longer than the rest`,
|
|
97413
|
+
"See [the reference documentation](https://example.com/docs/reference) for more detail on this API.",
|
|
97414
|
+
"\u65E5\u672C\u8A9E\u306E\u30C6\u30AD\u30B9\u30C8\u306F\u82F1\u8A9E\u3068\u6BD4\u3079\u3066\u30C8\u30FC\u30AF\u30F3\u3042\u305F\u308A\u306E\u6587\u5B57\u6570\u304C\u5927\u304D\u304F\u7570\u306A\u308B\u3053\u3068\u304C\u3042\u308A\u307E\u3059\u3002",
|
|
97415
|
+
"SELECT id, name, description FROM entries WHERE tags LIKE '%embedding%' ORDER BY updated_at DESC LIMIT 50;",
|
|
97416
|
+
"A longer paragraph mixing punctuation, numbers (like 42 and 3.14), and technical terms such as `tokenizer`, `embedding`, and `context window`."
|
|
97417
|
+
];
|
|
97418
|
+
function percentileIndex(length, percentile) {
|
|
97419
|
+
if (length <= 1)
|
|
97420
|
+
return 0;
|
|
97421
|
+
return Math.max(0, Math.min(length - 1, Math.floor((length - 1) * percentile)));
|
|
97450
97422
|
}
|
|
97451
|
-
|
|
97452
|
-
|
|
97453
|
-
|
|
97454
|
-
|
|
97455
|
-
|
|
97456
|
-
|
|
97457
|
-
|
|
97458
|
-
|
|
97459
|
-
|
|
97460
|
-
|
|
97423
|
+
async function calibrateCharsPerToken(countTokens) {
|
|
97424
|
+
const ratios = [];
|
|
97425
|
+
for (const text of CALIBRATION_SHAPES) {
|
|
97426
|
+
try {
|
|
97427
|
+
const tokens = await countTokens(text);
|
|
97428
|
+
if (tokens > 0)
|
|
97429
|
+
ratios.push(text.length / tokens);
|
|
97430
|
+
} catch {}
|
|
97431
|
+
}
|
|
97432
|
+
if (ratios.length === 0)
|
|
97433
|
+
return CHARS_PER_TOKEN_TAIL;
|
|
97434
|
+
ratios.sort((a, b) => a - b);
|
|
97435
|
+
return ratios[percentileIndex(ratios.length, CALIBRATION_PERCENTILE)];
|
|
97461
97436
|
}
|
|
97462
|
-
function
|
|
97463
|
-
|
|
97464
|
-
|
|
97465
|
-
|
|
97466
|
-
contended.cause = error2;
|
|
97467
|
-
return contended;
|
|
97437
|
+
function timedFetch(fetchImpl, url2, init, timeoutMs, externalSignal) {
|
|
97438
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
97439
|
+
const signal = externalSignal ? AbortSignal.any([externalSignal, timeoutSignal]) : timeoutSignal;
|
|
97440
|
+
return fetchImpl(url2, { ...init, signal });
|
|
97468
97441
|
}
|
|
97469
|
-
|
|
97470
|
-
|
|
97471
|
-
|
|
97472
|
-
|
|
97473
|
-
return
|
|
97442
|
+
async function readJson(res) {
|
|
97443
|
+
try {
|
|
97444
|
+
return await res.json();
|
|
97445
|
+
} catch {
|
|
97446
|
+
return;
|
|
97474
97447
|
}
|
|
97475
|
-
|
|
97476
|
-
|
|
97448
|
+
}
|
|
97449
|
+
async function probeLlamaCppCountTokens(origin, fetchImpl, timeoutMs, probeSignal) {
|
|
97450
|
+
const tokenizeOnce = (text, signal) => timedFetch(fetchImpl, `${origin}/tokenize`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ content: text }) }, timeoutMs, signal);
|
|
97451
|
+
const presence = await tokenizeOnce(TOKENIZE_PRESENCE_PROBE_TEXT, probeSignal);
|
|
97452
|
+
if (!presence.ok)
|
|
97453
|
+
return;
|
|
97454
|
+
const presenceBody = await readJson(presence);
|
|
97455
|
+
if (!Array.isArray(presenceBody?.tokens))
|
|
97456
|
+
return;
|
|
97457
|
+
return async (text) => {
|
|
97458
|
+
const res = await tokenizeOnce(text, undefined);
|
|
97459
|
+
if (!res.ok)
|
|
97460
|
+
throw new Error(`llama.cpp /tokenize request failed (${res.status})`);
|
|
97461
|
+
const json = await readJson(res);
|
|
97462
|
+
if (!Array.isArray(json?.tokens))
|
|
97463
|
+
throw new Error("Unexpected /tokenize response: missing tokens array");
|
|
97464
|
+
return json.tokens.length;
|
|
97465
|
+
};
|
|
97466
|
+
}
|
|
97467
|
+
async function probeLlamaCpp(origin, config, fetchImpl, timeoutMs, signal) {
|
|
97468
|
+
const res = await timedFetch(fetchImpl, `${origin}/props`, { method: "GET" }, timeoutMs, signal);
|
|
97469
|
+
if (!res.ok)
|
|
97470
|
+
return;
|
|
97471
|
+
const body = await readJson(res);
|
|
97472
|
+
const windowTokens = body?.default_generation_settings?.n_ctx;
|
|
97473
|
+
if (typeof windowTokens !== "number" || !Number.isFinite(windowTokens) || windowTokens <= 0)
|
|
97474
|
+
return;
|
|
97475
|
+
const probedSlots = typeof body?.total_slots === "number" && body.total_slots > 0 ? body.total_slots : 1;
|
|
97476
|
+
const slots = config.concurrency ?? probedSlots;
|
|
97477
|
+
const countTokens = await probeLlamaCppCountTokens(origin, fetchImpl, timeoutMs, signal).catch(() => {
|
|
97478
|
+
return;
|
|
97479
|
+
});
|
|
97480
|
+
const charsPerToken = countTokens ? await calibrateCharsPerToken(countTokens) : CHARS_PER_TOKEN_TAIL;
|
|
97481
|
+
return { windowTokens, slots, source: "llama.cpp", charsPerToken };
|
|
97482
|
+
}
|
|
97483
|
+
var OLLAMA_DEFAULT_SLOTS = 1;
|
|
97484
|
+
function findOllamaContextLength(modelInfo) {
|
|
97485
|
+
for (const [key, value] of Object.entries(modelInfo)) {
|
|
97486
|
+
if (key.endsWith(".context_length") && typeof value === "number" && Number.isFinite(value) && value > 0) {
|
|
97487
|
+
return value;
|
|
97488
|
+
}
|
|
97477
97489
|
}
|
|
97478
|
-
return
|
|
97490
|
+
return;
|
|
97479
97491
|
}
|
|
97480
|
-
function
|
|
97481
|
-
|
|
97492
|
+
async function probeOllama(origin, config, fetchImpl, timeoutMs, signal) {
|
|
97493
|
+
if (!config.model)
|
|
97494
|
+
return;
|
|
97495
|
+
const res = await timedFetch(fetchImpl, `${origin}/api/show`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: config.model }) }, timeoutMs, signal);
|
|
97496
|
+
if (!res.ok)
|
|
97497
|
+
return;
|
|
97498
|
+
const body = await readJson(res);
|
|
97499
|
+
if (!body?.model_info)
|
|
97500
|
+
return;
|
|
97501
|
+
const windowTokens = findOllamaContextLength(body.model_info);
|
|
97502
|
+
if (windowTokens === undefined)
|
|
97503
|
+
return;
|
|
97504
|
+
return {
|
|
97505
|
+
windowTokens,
|
|
97506
|
+
slots: config.concurrency ?? OLLAMA_DEFAULT_SLOTS,
|
|
97507
|
+
source: "ollama",
|
|
97508
|
+
charsPerToken: CHARS_PER_TOKEN_TAIL
|
|
97509
|
+
};
|
|
97482
97510
|
}
|
|
97483
|
-
|
|
97484
|
-
|
|
97485
|
-
|
|
97486
|
-
|
|
97487
|
-
|
|
97488
|
-
|
|
97489
|
-
|
|
97490
|
-
|
|
97491
|
-
|
|
97511
|
+
function resolveOrigin(endpoint) {
|
|
97512
|
+
if (!endpoint)
|
|
97513
|
+
return;
|
|
97514
|
+
try {
|
|
97515
|
+
const parsed = new URL(endpoint);
|
|
97516
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
|
|
97517
|
+
return;
|
|
97518
|
+
return parsed.origin;
|
|
97519
|
+
} catch {
|
|
97520
|
+
return;
|
|
97492
97521
|
}
|
|
97493
|
-
|
|
97494
|
-
|
|
97522
|
+
}
|
|
97523
|
+
function resolveProbeTimeoutMs(config) {
|
|
97524
|
+
return config.timeoutMs ?? HEALTH_PROBE_TIMEOUT_MS;
|
|
97525
|
+
}
|
|
97526
|
+
function defaultLimits(config) {
|
|
97495
97527
|
return {
|
|
97496
|
-
|
|
97497
|
-
|
|
97498
|
-
|
|
97528
|
+
windowTokens: DEFAULT_WINDOW_TOKENS,
|
|
97529
|
+
slots: config.concurrency ?? 1,
|
|
97530
|
+
source: "default",
|
|
97531
|
+
charsPerToken: CHARS_PER_TOKEN_TAIL
|
|
97499
97532
|
};
|
|
97500
97533
|
}
|
|
97501
|
-
function
|
|
97502
|
-
|
|
97503
|
-
const mid = Math.floor(sorted.length / 2);
|
|
97504
|
-
return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
|
|
97534
|
+
function isUsableWindow(limits) {
|
|
97535
|
+
return unitMaxChars(limits) > 0;
|
|
97505
97536
|
}
|
|
97537
|
+
async function probeProviderLimitsUncached(config, opts) {
|
|
97538
|
+
const origin = resolveOrigin(config.endpoint);
|
|
97539
|
+
if (!origin)
|
|
97540
|
+
return defaultLimits(config);
|
|
97541
|
+
const fetchImpl = opts?.fetch ?? fetch;
|
|
97542
|
+
const timeoutMs = resolveProbeTimeoutMs(config);
|
|
97543
|
+
const signal = opts?.signal;
|
|
97544
|
+
const llamaCpp = await probeLlamaCpp(origin, config, fetchImpl, timeoutMs, signal).catch(() => {
|
|
97545
|
+
return;
|
|
97546
|
+
});
|
|
97547
|
+
if (llamaCpp && isUsableWindow(llamaCpp))
|
|
97548
|
+
return llamaCpp;
|
|
97549
|
+
const ollama = await probeOllama(origin, config, fetchImpl, timeoutMs, signal).catch(() => {
|
|
97550
|
+
return;
|
|
97551
|
+
});
|
|
97552
|
+
if (ollama && isUsableWindow(ollama))
|
|
97553
|
+
return ollama;
|
|
97554
|
+
return defaultLimits(config);
|
|
97555
|
+
}
|
|
97556
|
+
var providerLimitsCache = new Map;
|
|
97557
|
+
async function probeProviderLimits(config, opts) {
|
|
97558
|
+
const cacheKey = JSON.stringify({
|
|
97559
|
+
endpoint: config.endpoint,
|
|
97560
|
+
model: config.model,
|
|
97561
|
+
concurrency: config.concurrency,
|
|
97562
|
+
timeoutMs: config.timeoutMs
|
|
97563
|
+
});
|
|
97564
|
+
const cached = providerLimitsCache.get(cacheKey);
|
|
97565
|
+
if (cached)
|
|
97566
|
+
return cached;
|
|
97567
|
+
const probe = probeProviderLimitsUncached(config, opts);
|
|
97568
|
+
providerLimitsCache.set(cacheKey, probe);
|
|
97569
|
+
return probe;
|
|
97570
|
+
}
|
|
97571
|
+
function unitMaxChars(limits) {
|
|
97572
|
+
return Math.max(0, Math.floor((limits.windowTokens - UNIT_HEADER_MARGIN_TOKENS) * limits.charsPerToken));
|
|
97573
|
+
}
|
|
97574
|
+
|
|
97575
|
+
// src/indexer/embedding-identity.ts
|
|
97506
97576
|
function deriveObservedEmbeddingIdentity(embedding, observedModel, observedVectorLen) {
|
|
97507
97577
|
if (isDeterministicEmbedEnabled()) {
|
|
97508
97578
|
return `deterministic:${DETERMINISTIC_EMBED_MODEL_ID}`;
|
|
@@ -97514,600 +97584,315 @@ function deriveObservedEmbeddingIdentity(embedding, observedModel, observedVecto
|
|
|
97514
97584
|
}
|
|
97515
97585
|
return `local:${embedding?.localModel ?? DEFAULT_LOCAL_MODEL}|${observedVectorLen}`;
|
|
97516
97586
|
}
|
|
97517
|
-
|
|
97518
|
-
|
|
97519
|
-
|
|
97520
|
-
|
|
97521
|
-
|
|
97522
|
-
|
|
97523
|
-
|
|
97524
|
-
|
|
97525
|
-
|
|
97526
|
-
|
|
97527
|
-
|
|
97528
|
-
observedModel = model;
|
|
97529
|
-
});
|
|
97530
|
-
} catch (error2) {
|
|
97531
|
-
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
97532
|
-
return {
|
|
97533
|
-
outcome: "unverifiable",
|
|
97534
|
-
message: `could not verify embedding compatibility (${message}); keeping existing vectors \u2014 rerun akm index when the endpoint is reachable`
|
|
97535
|
-
};
|
|
97536
|
-
}
|
|
97537
|
-
const observedVectorLen = canaryVectors.find((vector) => vector !== undefined)?.length;
|
|
97538
|
-
const observedIdentity = deriveObservedEmbeddingIdentity(config.embedding, observedModel, observedVectorLen);
|
|
97539
|
-
const storedIdentity = getMeta(db, "embeddingIdentity");
|
|
97540
|
-
if (storedIdentity && observedIdentity && storedIdentity === observedIdentity) {
|
|
97541
|
-
return { outcome: "keep", verified: true, identity: observedIdentity, viaIdentityMatch: true };
|
|
97542
|
-
}
|
|
97543
|
-
const pairs = samples.map((sample, i) => ({ stored: sample.vector, fresh: canaryVectors[i] }));
|
|
97544
|
-
const decision = decideEmbeddingCompatibility(pairs);
|
|
97545
|
-
if (decision.outcome === "unverifiable") {
|
|
97546
|
-
const message = skips[0]?.message ?? "embedding provider returned no vectors for the canary sample";
|
|
97547
|
-
return {
|
|
97548
|
-
outcome: "unverifiable",
|
|
97549
|
-
message: `could not verify embedding compatibility (${message}); keeping existing vectors \u2014 rerun akm index when the endpoint is reachable`
|
|
97550
|
-
};
|
|
97551
|
-
}
|
|
97552
|
-
if (decision.outcome === "keep") {
|
|
97553
|
-
return {
|
|
97554
|
-
outcome: "keep",
|
|
97555
|
-
verified: true,
|
|
97556
|
-
identity: observedIdentity,
|
|
97557
|
-
viaIdentityMatch: false,
|
|
97558
|
-
medianSimilarity: decision.medianSimilarity
|
|
97559
|
-
};
|
|
97560
|
-
}
|
|
97561
|
-
return {
|
|
97562
|
-
outcome: "rebuild",
|
|
97563
|
-
identity: observedIdentity,
|
|
97564
|
-
reason: `vectors differ (median similarity ${decision.medianSimilarity?.toFixed(3)})`
|
|
97565
|
-
};
|
|
97587
|
+
|
|
97588
|
+
// src/indexer/drain.ts
|
|
97589
|
+
var CIRCUIT_BREAKER_THRESHOLD = 3;
|
|
97590
|
+
var CIRCUIT_BREAKER_WINDOW = CIRCUIT_BREAKER_THRESHOLD * 2;
|
|
97591
|
+
function pushBreakerOutcome(window2, isFailure) {
|
|
97592
|
+
window2.push(isFailure);
|
|
97593
|
+
if (window2.length > CIRCUIT_BREAKER_WINDOW)
|
|
97594
|
+
window2.shift();
|
|
97595
|
+
}
|
|
97596
|
+
function breakerFailureCount(window2) {
|
|
97597
|
+
return window2.reduce((n, isFailure) => n + (isFailure ? 1 : 0), 0);
|
|
97566
97598
|
}
|
|
97599
|
+
var DRAIN_BATCH_PROGRESS_PREFIX = "[drain] batch ";
|
|
97567
97600
|
function throwIfAborted(signal) {
|
|
97568
97601
|
if (signal?.aborted) {
|
|
97569
|
-
throw signal.reason instanceof Error ? signal.reason : new Error("
|
|
97602
|
+
throw signal.reason instanceof Error ? signal.reason : new Error("drain interrupted");
|
|
97570
97603
|
}
|
|
97571
97604
|
}
|
|
97572
|
-
|
|
97573
|
-
|
|
97574
|
-
|
|
97605
|
+
function selectAllUnitHashes(db) {
|
|
97606
|
+
return db.prepare("SELECT unit_hash FROM unit_texts ORDER BY unit_hash").all().map((row) => row.unit_hash);
|
|
97607
|
+
}
|
|
97608
|
+
function fetchUnitTexts(db, hashes) {
|
|
97609
|
+
const texts = new Map;
|
|
97610
|
+
for (let offset = 0;offset < hashes.length; offset += SQLITE_CHUNK_SIZE) {
|
|
97611
|
+
const chunk2 = hashes.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
97612
|
+
const placeholders = chunk2.map(() => "?").join(",");
|
|
97613
|
+
const rows = db.prepare(`SELECT unit_hash, text FROM unit_texts WHERE unit_hash IN (${placeholders})`).all(...chunk2);
|
|
97614
|
+
for (const row of rows)
|
|
97615
|
+
texts.set(row.unit_hash, row.text);
|
|
97575
97616
|
}
|
|
97576
|
-
|
|
97617
|
+
return texts;
|
|
97618
|
+
}
|
|
97619
|
+
async function resolveEmbeddingPacking(config, signal) {
|
|
97620
|
+
const base3 = config.embedding ?? {};
|
|
97621
|
+
const limits = await probeProviderLimits(base3, { signal });
|
|
97622
|
+
return {
|
|
97623
|
+
embeddingConfig: { ...base3, concurrency: base3.concurrency ?? limits.slots },
|
|
97624
|
+
packing: {
|
|
97625
|
+
tokenBudget: limits.windowTokens,
|
|
97626
|
+
charsPerToken: limits.charsPerToken,
|
|
97627
|
+
windowIsKnown: limits.source !== "default",
|
|
97628
|
+
ollamaNumCtx: limits.source === "ollama" ? limits.windowTokens : undefined
|
|
97629
|
+
}
|
|
97630
|
+
};
|
|
97631
|
+
}
|
|
97632
|
+
function emitCredentialDiagnostic(config, onProgress) {
|
|
97633
|
+
if (!onProgress || !hasRemoteEndpoint(config.embedding ?? {}))
|
|
97634
|
+
return;
|
|
97635
|
+
const endpoint = normalizeEmbeddingEndpoint(config.embedding?.endpoint ?? "");
|
|
97636
|
+
const credential = describeEmbeddingCredential(config.embedding?.apiKey);
|
|
97637
|
+
const configFileSuffix = isVerbose() ? `; config: ${getConfigPath()}` : "";
|
|
97638
|
+
onProgress(`[embed] endpoint ${endpoint}, model ${config.embedding?.model ?? "unknown"}; credential: ${credential}${configFileSuffix}`);
|
|
97639
|
+
}
|
|
97640
|
+
function formatDoneLine(counts) {
|
|
97641
|
+
return `[drain] done: ${counts.pending} pending, ${counts.embedded} embedded, ${counts.failed} failed, ` + `${counts.skipped} skipped (identity: ${counts.identity ?? "unknown"})`;
|
|
97642
|
+
}
|
|
97643
|
+
async function drainEmbeddingQueue(db, config, opts = {}) {
|
|
97644
|
+
throwIfAborted(opts.signal);
|
|
97645
|
+
let identity3 = getMeta(db, "embeddingIdentity") ?? null;
|
|
97577
97646
|
if (config.semanticSearchMode === "off") {
|
|
97578
|
-
|
|
97579
|
-
|
|
97580
|
-
|
|
97581
|
-
|
|
97582
|
-
|
|
97583
|
-
|
|
97584
|
-
|
|
97585
|
-
|
|
97586
|
-
|
|
97587
|
-
|
|
97588
|
-
|
|
97589
|
-
|
|
97590
|
-
|
|
97591
|
-
|
|
97592
|
-
|
|
97593
|
-
const
|
|
97594
|
-
|
|
97595
|
-
|
|
97596
|
-
const
|
|
97597
|
-
if (
|
|
97598
|
-
|
|
97599
|
-
|
|
97600
|
-
|
|
97601
|
-
|
|
97602
|
-
|
|
97603
|
-
|
|
97604
|
-
|
|
97605
|
-
|
|
97606
|
-
|
|
97607
|
-
|
|
97608
|
-
|
|
97609
|
-
|
|
97610
|
-
|
|
97611
|
-
|
|
97612
|
-
|
|
97613
|
-
|
|
97614
|
-
|
|
97615
|
-
|
|
97616
|
-
purgeEmbeddings(db, { dropVecTable: true });
|
|
97617
|
-
purgeEmbeddingSalvage(db);
|
|
97618
|
-
deleteMeta(db, "embeddingDim");
|
|
97619
|
-
setMeta(db, "embeddingFingerprint", currentFingerprint);
|
|
97620
|
-
if (decision.identity)
|
|
97621
|
-
setMeta(db, "embeddingIdentity", decision.identity);
|
|
97622
|
-
else
|
|
97623
|
-
deleteMeta(db, "embeddingIdentity");
|
|
97624
|
-
})();
|
|
97625
|
-
targetEntryIds = undefined;
|
|
97626
|
-
rebuildReason = decision.reason;
|
|
97627
|
-
} else {
|
|
97628
|
-
setMeta(db, "embeddingFingerprint", currentFingerprint);
|
|
97629
|
-
relabelEmbeddingSalvageFingerprint(db, storedFingerprint, currentFingerprint);
|
|
97630
|
-
if (decision.identity)
|
|
97631
|
-
setMeta(db, "embeddingIdentity", decision.identity);
|
|
97632
|
-
if (decision.verified) {
|
|
97633
|
-
const keptCount = getEmbeddingCount(db);
|
|
97634
|
-
const detail = decision.viaIdentityMatch ? "server-reported model unchanged" : `stored vectors are compatible (median similarity ${decision.medianSimilarity?.toFixed(3)})`;
|
|
97635
|
-
const message = `[embed] embedding model renamed (${storedFingerprint} \u2192 ${currentFingerprint}); ${detail}, keeping ${keptCount} embedding${keptCount === 1 ? "" : "s"}.`;
|
|
97636
|
-
warn(message);
|
|
97637
|
-
onProgress({ phase: "embeddings", message });
|
|
97638
|
-
}
|
|
97647
|
+
return { pending: 0, embedded: 0, failed: 0, skipped: 0, identity: identity3 };
|
|
97648
|
+
}
|
|
97649
|
+
const candidateHashes = opts.onlyHashes ? [...new Set(opts.onlyHashes)] : selectAllUnitHashes(db);
|
|
97650
|
+
const missingHashes = identity3 ? listMissingHashes(db, candidateHashes, identity3) : candidateHashes;
|
|
97651
|
+
const pending = missingHashes.length;
|
|
97652
|
+
const emitDone = (counts) => {
|
|
97653
|
+
opts.onProgress?.(formatDoneLine(counts));
|
|
97654
|
+
return counts;
|
|
97655
|
+
};
|
|
97656
|
+
if (!isVecAvailable(db)) {
|
|
97657
|
+
return emitDone({ pending, embedded: 0, failed: 0, skipped: pending, identity: identity3 });
|
|
97658
|
+
}
|
|
97659
|
+
if (pending === 0) {
|
|
97660
|
+
return emitDone({ pending: 0, embedded: 0, failed: 0, skipped: 0, identity: identity3 });
|
|
97661
|
+
}
|
|
97662
|
+
const boundedHashes = opts.limit !== undefined ? missingHashes.slice(0, opts.limit) : missingHashes;
|
|
97663
|
+
const textByHash = fetchUnitTexts(db, boundedHashes);
|
|
97664
|
+
const orderedHashes = boundedHashes.filter((hash4) => textByHash.has(hash4));
|
|
97665
|
+
const texts = orderedHashes.map((hash4) => textByHash.get(hash4));
|
|
97666
|
+
if (texts.length === 0) {
|
|
97667
|
+
return emitDone({ pending, embedded: 0, failed: 0, skipped: 0, identity: identity3 });
|
|
97668
|
+
}
|
|
97669
|
+
emitCredentialDiagnostic(config, opts.onProgress);
|
|
97670
|
+
const { embeddingConfig, packing } = await resolveEmbeddingPacking(config, opts.signal);
|
|
97671
|
+
let embedded = 0;
|
|
97672
|
+
let failed = 0;
|
|
97673
|
+
let batchNumber = 0;
|
|
97674
|
+
const singleDocFailureWindow = [];
|
|
97675
|
+
const networkErrorFailureWindow = [];
|
|
97676
|
+
let identityDecidedThisCall = false;
|
|
97677
|
+
const onSkip = (skip) => {
|
|
97678
|
+
failed++;
|
|
97679
|
+
if (!skip.batchStart)
|
|
97680
|
+
return;
|
|
97681
|
+
if (skip.reason === "context-window-exceeded") {
|
|
97682
|
+
singleDocFailureWindow.length = 0;
|
|
97683
|
+
networkErrorFailureWindow.length = 0;
|
|
97684
|
+
return;
|
|
97639
97685
|
}
|
|
97640
|
-
|
|
97641
|
-
|
|
97642
|
-
|
|
97643
|
-
|
|
97644
|
-
throwIfAborted(signal);
|
|
97645
|
-
const allEntries = getAllEntriesForEmbedding(db, targetEntryIds);
|
|
97646
|
-
let vecFailedCount = 0;
|
|
97647
|
-
let vecUnavailableCount = 0;
|
|
97648
|
-
const { reusedCount, remaining: candidateEntries } = reuseSalvagedEmbeddings(db, allEntries, currentFingerprint, (entry, embedding) => {
|
|
97649
|
-
const result = upsertEmbedding(db, entry.id, embedding);
|
|
97650
|
-
if (result.vec === "failed")
|
|
97651
|
-
vecFailedCount++;
|
|
97652
|
-
if (result.vec === "unavailable")
|
|
97653
|
-
vecUnavailableCount++;
|
|
97654
|
-
return result.stored;
|
|
97655
|
-
});
|
|
97656
|
-
if (reusedCount > 0) {
|
|
97657
|
-
onProgress({
|
|
97658
|
-
phase: "embeddings",
|
|
97659
|
-
message: `Reused ${reusedCount} embedding${reusedCount === 1 ? "" : "s"} from the previous generation; embedding ${candidateEntries.length} new.`
|
|
97660
|
-
});
|
|
97686
|
+
pushBreakerOutcome(singleDocFailureWindow, skip.batchSize === 1);
|
|
97687
|
+
pushBreakerOutcome(networkErrorFailureWindow, skip.failureKind === "network-error");
|
|
97688
|
+
if (breakerFailureCount(singleDocFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD || breakerFailureCount(networkErrorFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD) {
|
|
97689
|
+
return false;
|
|
97661
97690
|
}
|
|
97662
|
-
|
|
97663
|
-
|
|
97664
|
-
|
|
97665
|
-
|
|
97666
|
-
|
|
97667
|
-
|
|
97668
|
-
|
|
97669
|
-
|
|
97670
|
-
|
|
97671
|
-
}
|
|
97672
|
-
let truncatedCount = 0;
|
|
97673
|
-
const texts = [];
|
|
97674
|
-
const pendingEntries = [];
|
|
97675
|
-
for (const entry of candidateEntries) {
|
|
97676
|
-
const capped = capEmbeddingText(entry.searchText, maxInputTokens);
|
|
97677
|
-
if (capped.text.length === 0)
|
|
97691
|
+
return;
|
|
97692
|
+
};
|
|
97693
|
+
const onBatch = (indices, embeddings, model, outcome) => {
|
|
97694
|
+
if (outcome?.outcome === "retrying" || outcome?.outcome === "budget-lowered")
|
|
97695
|
+
return;
|
|
97696
|
+
const rows = [];
|
|
97697
|
+
for (let k2 = 0;k2 < indices.length; k2++) {
|
|
97698
|
+
const embedding = embeddings[k2];
|
|
97699
|
+
if (!embedding)
|
|
97678
97700
|
continue;
|
|
97679
|
-
|
|
97680
|
-
|
|
97681
|
-
|
|
97682
|
-
|
|
97683
|
-
|
|
97684
|
-
|
|
97685
|
-
|
|
97686
|
-
onProgress({ phase: "embeddings", message });
|
|
97687
|
-
}
|
|
97688
|
-
if (rebuildReason) {
|
|
97689
|
-
const message = `[embed] Re-embedding ${pendingEntries.length} entr${pendingEntries.length === 1 ? "y" : "ies"} because ${rebuildReason}`;
|
|
97690
|
-
onProgress({ phase: "embeddings", message });
|
|
97691
|
-
}
|
|
97692
|
-
onProgress({
|
|
97693
|
-
phase: "embeddings",
|
|
97694
|
-
message: `Generating embeddings for ${pendingEntries.length} entr${pendingEntries.length === 1 ? "y" : "ies"}.`
|
|
97695
|
-
});
|
|
97696
|
-
if (isVerbose()) {
|
|
97697
|
-
if (hasRemoteEndpoint(config.embedding ?? {})) {
|
|
97698
|
-
const tokenBudget = config.embedding?.maxTokens ?? DEFAULT_TOKEN_BUDGET;
|
|
97699
|
-
const maxCount = config.embedding?.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
|
|
97700
|
-
const batches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
|
|
97701
|
-
const batchNumberByIndex = new Map;
|
|
97702
|
-
batches.forEach((batch, batchIdx) => {
|
|
97703
|
-
for (const i of batch.indices)
|
|
97704
|
-
batchNumberByIndex.set(i, batchIdx + 1);
|
|
97705
|
-
});
|
|
97706
|
-
for (const [i, entry] of pendingEntries.entries()) {
|
|
97707
|
-
const chars = entry.searchText.length;
|
|
97708
|
-
const tokens = estimateTokenCount(entry.searchText);
|
|
97709
|
-
const batch = batches[batchNumberByIndex.get(i) - 1];
|
|
97710
|
-
const label = batch?.oversized ? "oversized (skipped)" : `batch ${batchNumberByIndex.get(i)}/${batches.length}`;
|
|
97711
|
-
warnVerbose(`[embed] ${entry.itemRef} (${chars} chars, est. ${tokens} tokens) \u2192 ${label}`);
|
|
97712
|
-
}
|
|
97713
|
-
} else {
|
|
97714
|
-
for (const entry of pendingEntries) {
|
|
97715
|
-
warnVerbose(`[embed] ${entry.itemRef} (${entry.searchText.length} chars, est. ${estimateTokenCount(entry.searchText)} tokens)`);
|
|
97701
|
+
const learned = deriveObservedEmbeddingIdentity(config.embedding, model, embedding.length);
|
|
97702
|
+
if (!identityDecidedThisCall) {
|
|
97703
|
+
identityDecidedThisCall = true;
|
|
97704
|
+
if (learned && learned !== identity3) {
|
|
97705
|
+
identity3 = learned;
|
|
97706
|
+
setMeta(db, "embeddingIdentity", identity3);
|
|
97707
|
+
dropOtherIdentities(db, identity3, embedding.length);
|
|
97716
97708
|
}
|
|
97717
97709
|
}
|
|
97710
|
+
const currentIdentity = identity3;
|
|
97711
|
+
if (currentIdentity === null || learned !== currentIdentity) {
|
|
97712
|
+
continue;
|
|
97713
|
+
}
|
|
97714
|
+
const hash4 = orderedHashes[indices[k2]];
|
|
97715
|
+
if (hash4)
|
|
97716
|
+
rows.push({ hash: hash4, identity: currentIdentity, vector: embedding });
|
|
97718
97717
|
}
|
|
97719
|
-
let
|
|
97720
|
-
|
|
97721
|
-
|
|
97722
|
-
|
|
97723
|
-
|
|
97724
|
-
|
|
97725
|
-
|
|
97726
|
-
|
|
97727
|
-
|
|
97728
|
-
|
|
97729
|
-
});
|
|
97730
|
-
}, 15000);
|
|
97731
|
-
const skips = [];
|
|
97732
|
-
const embedStart = Date.now();
|
|
97733
|
-
let consecutiveSingleDocFailures = 0;
|
|
97734
|
-
let consecutiveNetworkErrorFailures = 0;
|
|
97735
|
-
let circuitBreakerReason;
|
|
97736
|
-
const onSkip = (skip) => {
|
|
97737
|
-
skips.push(skip);
|
|
97738
|
-
if (!skip.batchStart)
|
|
97739
|
-
return;
|
|
97740
|
-
if (skip.reason === "context-window-exceeded") {
|
|
97741
|
-
consecutiveSingleDocFailures = 0;
|
|
97742
|
-
consecutiveNetworkErrorFailures = 0;
|
|
97743
|
-
return;
|
|
97744
|
-
}
|
|
97745
|
-
consecutiveSingleDocFailures = skip.batchSize === 1 ? consecutiveSingleDocFailures + 1 : 0;
|
|
97746
|
-
consecutiveNetworkErrorFailures = skip.failureKind === "network-error" ? consecutiveNetworkErrorFailures + 1 : 0;
|
|
97747
|
-
if (consecutiveSingleDocFailures >= CIRCUIT_BREAKER_THRESHOLD || consecutiveNetworkErrorFailures >= CIRCUIT_BREAKER_THRESHOLD) {
|
|
97748
|
-
circuitBreakerReason = skip.message;
|
|
97749
|
-
return false;
|
|
97750
|
-
}
|
|
97751
|
-
return;
|
|
97752
|
-
};
|
|
97753
|
-
let observedModel;
|
|
97754
|
-
let observedVectorLen;
|
|
97755
|
-
const reportPerBatchLine = hasRemoteEndpoint(config.embedding ?? {});
|
|
97756
|
-
const onBatch = (indices, batchEmbeddings, model, outcome) => {
|
|
97757
|
-
if (outcome?.outcome === "retrying") {
|
|
97758
|
-
if (reportPerBatchLine) {
|
|
97759
|
-
onProgress({
|
|
97760
|
-
phase: "embeddings",
|
|
97761
|
-
message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens \u2192 retrying after ${(outcome.elapsedMs / 1000).toFixed(1)} s`
|
|
97762
|
-
});
|
|
97763
|
-
}
|
|
97764
|
-
return;
|
|
97765
|
-
}
|
|
97766
|
-
if (outcome?.outcome === "budget-lowered") {
|
|
97767
|
-
if (reportPerBatchLine) {
|
|
97768
|
-
onProgress({
|
|
97769
|
-
phase: "embeddings",
|
|
97770
|
-
message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens \u2192 ${outcome.reason}`
|
|
97771
|
-
});
|
|
97772
|
-
}
|
|
97773
|
-
return;
|
|
97774
|
-
}
|
|
97775
|
-
if (model)
|
|
97776
|
-
observedModel = model;
|
|
97777
|
-
if (batchEmbeddings.some((embedding) => embedding !== undefined)) {
|
|
97778
|
-
consecutiveSingleDocFailures = 0;
|
|
97779
|
-
consecutiveNetworkErrorFailures = 0;
|
|
97780
|
-
}
|
|
97781
|
-
db.transaction(() => {
|
|
97782
|
-
for (let k2 = 0;k2 < indices.length; k2++) {
|
|
97783
|
-
const index = indices[k2];
|
|
97784
|
-
const entry = pendingEntries[index];
|
|
97785
|
-
if (!entry)
|
|
97786
|
-
continue;
|
|
97787
|
-
const embedding = batchEmbeddings[k2];
|
|
97788
|
-
if (!embedding) {
|
|
97789
|
-
embedFailedCount++;
|
|
97790
|
-
continue;
|
|
97791
|
-
}
|
|
97792
|
-
if (observedVectorLen === undefined)
|
|
97793
|
-
observedVectorLen = embedding.length;
|
|
97794
|
-
const result = upsertEmbedding(db, entry.id, embedding);
|
|
97795
|
-
if (result.stored) {
|
|
97796
|
-
storedCount++;
|
|
97797
|
-
storedTokens += estimateTokenCount(texts[index]);
|
|
97798
|
-
} else {
|
|
97799
|
-
skippedCount++;
|
|
97800
|
-
}
|
|
97801
|
-
if (result.vec === "failed")
|
|
97802
|
-
vecFailedCount++;
|
|
97803
|
-
if (result.vec === "unavailable")
|
|
97804
|
-
vecUnavailableCount++;
|
|
97805
|
-
}
|
|
97806
|
-
})();
|
|
97807
|
-
if (outcome && outcome.reason !== "oversized" && reportPerBatchLine) {
|
|
97808
|
-
const elapsedSeconds2 = (outcome.elapsedMs / 1000).toFixed(1);
|
|
97809
|
-
const outcomeLabel = outcome.outcome === "stored" ? `${outcome.docCount} stored (${elapsedSeconds2} s)` : `failed: ${outcome.reason}`;
|
|
97810
|
-
onProgress({
|
|
97811
|
-
phase: "embeddings",
|
|
97812
|
-
message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens \u2192 ${outcomeLabel}`
|
|
97813
|
-
});
|
|
97718
|
+
let storageBreakerTripped = false;
|
|
97719
|
+
if (rows.length > 0) {
|
|
97720
|
+
const result = upsertUnitVectors(db, rows);
|
|
97721
|
+
embedded += result.inserted;
|
|
97722
|
+
failed += result.failed;
|
|
97723
|
+
if (result.failed > 0) {
|
|
97724
|
+
pushBreakerOutcome(singleDocFailureWindow, true);
|
|
97725
|
+
pushBreakerOutcome(networkErrorFailureWindow, true);
|
|
97726
|
+
if (breakerFailureCount(singleDocFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD || breakerFailureCount(networkErrorFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD) {
|
|
97727
|
+
storageBreakerTripped = true;
|
|
97814
97728
|
}
|
|
97815
|
-
onProgress({
|
|
97816
|
-
phase: "embeddings",
|
|
97817
|
-
message: `Embedded ${storedCount}/${pendingEntries.length} entries.`
|
|
97818
|
-
});
|
|
97819
|
-
};
|
|
97820
|
-
await embedBatch(texts, config.embedding, signal, onSkip, onBatch);
|
|
97821
|
-
throwIfAborted(signal);
|
|
97822
|
-
const elapsedSeconds = Math.max((Date.now() - embedStart) / 1000, 0.001);
|
|
97823
|
-
if (skippedCount > 0) {
|
|
97824
|
-
warn(`[embed] ${skippedCount} embedding${skippedCount === 1 ? "" : "s"} skipped (entry deleted between queue and write)`);
|
|
97825
|
-
}
|
|
97826
|
-
const vecGenerationComplete = targetEntryIds === undefined ? isVecFastPathComplete(db) : vecFastPathWasReady;
|
|
97827
|
-
setVecFastPathReady(db, vecFailedCount === 0 && vecUnavailableCount === 0 && vecGenerationComplete);
|
|
97828
|
-
if (vecFailedCount > 0) {
|
|
97829
|
-
warn(`[embed] ${vecFailedCount} sqlite-vec fast-path insert${vecFailedCount === 1 ? "" : "s"} failed \u2014 ` + "semantic search will use the slower JS-cosine fallback over stored embeddings. " + "Rebuild with 'akm index --full' after resolving the vec table (often a vector-dimension mismatch).");
|
|
97830
|
-
}
|
|
97831
|
-
const entriesPerSec = storedCount / elapsedSeconds;
|
|
97832
|
-
const tokensPerSec = storedTokens / elapsedSeconds;
|
|
97833
|
-
const totalStored = storedCount + reusedCount;
|
|
97834
|
-
const oversizedSkips = skips.filter((skip) => skip.reason === "context-window-exceeded");
|
|
97835
|
-
const timedOutSkips = skips.filter((skip) => skip.reason === "batch-request-failed" && skip.failureKind === "timeout");
|
|
97836
|
-
const failedSkips = skips.filter((skip) => skip.reason === "batch-request-failed" && skip.failureKind !== "timeout");
|
|
97837
|
-
const throughputLine = reusedCount > 0 ? `Stored ${totalStored} embedding${totalStored === 1 ? "" : "s"} (${reusedCount} reused, ${storedCount} newly embedded) in ${elapsedSeconds.toFixed(1)}s (${entriesPerSec.toFixed(1)} entries/s, ~${Math.round(tokensPerSec)} tokens/s)` : `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"} in ${elapsedSeconds.toFixed(1)}s (${entriesPerSec.toFixed(1)} entries/s, ~${Math.round(tokensPerSec)} tokens/s)`;
|
|
97838
|
-
onProgress({
|
|
97839
|
-
phase: "embeddings",
|
|
97840
|
-
message: `${throughputLine}; ${oversizedSkips.length} oversized skipped, ${timedOutSkips.length} timed out, ${failedSkips.length} failed.`
|
|
97841
|
-
});
|
|
97842
|
-
const printSkipList = (label, skipList) => {
|
|
97843
|
-
if (skipList.length === 0)
|
|
97844
|
-
return;
|
|
97845
|
-
const limit = isVerbose() ? skipList.length : 20;
|
|
97846
|
-
const listed = skipList.slice(0, limit).map((skip) => ` - ${pendingEntries[skip.index]?.itemRef ?? skip.index}: ${skip.message}`).join(`
|
|
97847
|
-
`);
|
|
97848
|
-
const more = skipList.length > limit ? `
|
|
97849
|
-
...and ${skipList.length - limit} more` : "";
|
|
97850
|
-
onProgress({ phase: "embeddings", message: `[embed] ${label} skipped:
|
|
97851
|
-
${listed}${more}` });
|
|
97852
|
-
};
|
|
97853
|
-
printSkipList("oversized documents", oversizedSkips);
|
|
97854
|
-
printSkipList("timed-out documents", timedOutSkips);
|
|
97855
|
-
printSkipList("failed documents", failedSkips);
|
|
97856
|
-
setMeta(db, "embeddingFingerprint", currentFingerprint);
|
|
97857
|
-
const observedIdentity = deriveObservedEmbeddingIdentity(config.embedding, observedModel, observedVectorLen);
|
|
97858
|
-
if (observedIdentity)
|
|
97859
|
-
setMeta(db, "embeddingIdentity", observedIdentity);
|
|
97860
|
-
if (circuitBreakerReason !== undefined) {
|
|
97861
|
-
const message = `embedding provider failed ${CIRCUIT_BREAKER_THRESHOLD} consecutive batches ` + `(last: ${circuitBreakerReason}); stopped after ${storedCount} embedding${storedCount === 1 ? "" : "s"} ` + "were stored \u2014 rerun akm index when the endpoint is healthy";
|
|
97862
|
-
warn(`[embed] ${message}`);
|
|
97863
|
-
onProgress({ phase: "embeddings", message });
|
|
97864
|
-
return { success: false, message, vecInsertFailures: vecFailedCount };
|
|
97865
|
-
}
|
|
97866
|
-
if (storedCount === 0 && embedFailedCount > 0) {
|
|
97867
|
-
const firstMessage = skips[0]?.message ?? "All embeddings failed.";
|
|
97868
|
-
return {
|
|
97869
|
-
success: false,
|
|
97870
|
-
message: `All ${embedFailedCount} embedding batch(es) failed: ${firstMessage}`
|
|
97871
|
-
};
|
|
97872
97729
|
}
|
|
97873
|
-
purgeEmbeddingSalvage(db);
|
|
97874
|
-
return { success: true, vecInsertFailures: vecFailedCount };
|
|
97875
|
-
} finally {
|
|
97876
|
-
if (heartbeatTimer)
|
|
97877
|
-
clearInterval(heartbeatTimer);
|
|
97878
97730
|
}
|
|
97879
|
-
|
|
97880
|
-
|
|
97881
|
-
|
|
97882
|
-
|
|
97883
|
-
|
|
97884
|
-
|
|
97885
|
-
|
|
97886
|
-
|
|
97887
|
-
|
|
97888
|
-
|
|
97889
|
-
|
|
97890
|
-
|
|
97891
|
-
|
|
97892
|
-
|
|
97893
|
-
|
|
97894
|
-
|
|
97895
|
-
const
|
|
97896
|
-
const
|
|
97897
|
-
|
|
97898
|
-
setMeta(db, "hasEmbeddings", ready ? "1" : "0");
|
|
97731
|
+
if (embeddings.some((embedding) => embedding !== undefined)) {
|
|
97732
|
+
pushBreakerOutcome(singleDocFailureWindow, false);
|
|
97733
|
+
pushBreakerOutcome(networkErrorFailureWindow, false);
|
|
97734
|
+
}
|
|
97735
|
+
batchNumber++;
|
|
97736
|
+
if (opts.onProgress) {
|
|
97737
|
+
const docCount = outcome?.docCount ?? indices.length;
|
|
97738
|
+
const label = outcome && outcome.outcome !== "stored" ? `failed: ${outcome.reason ?? "unknown"}` : `${rows.length} stored`;
|
|
97739
|
+
opts.onProgress(`${DRAIN_BATCH_PROGRESS_PREFIX}${batchNumber}: ${docCount} docs \u2192 ${label}`);
|
|
97740
|
+
}
|
|
97741
|
+
if (storageBreakerTripped) {
|
|
97742
|
+
throw new Error(`Circuit breaker: ${CIRCUIT_BREAKER_THRESHOLD} storage write failures while embedding; stopping further provider requests this call.`);
|
|
97743
|
+
}
|
|
97744
|
+
};
|
|
97745
|
+
await embedBatch(texts, embeddingConfig, opts.signal, onSkip, onBatch, packing);
|
|
97746
|
+
throwIfAborted(opts.signal);
|
|
97747
|
+
const attempted = texts.length;
|
|
97748
|
+
const skipped = Math.max(0, attempted - embedded - failed);
|
|
97749
|
+
return emitDone({ pending, embedded, failed, skipped, identity: identity3 });
|
|
97899
97750
|
}
|
|
97900
97751
|
|
|
97901
|
-
// src/indexer/
|
|
97902
|
-
|
|
97903
|
-
|
|
97904
|
-
// src/indexer/scan/drain-dir.ts
|
|
97905
|
-
import path63 from "path";
|
|
97752
|
+
// src/indexer/reconcile.ts
|
|
97753
|
+
import fs51 from "fs";
|
|
97754
|
+
import path62 from "path";
|
|
97906
97755
|
init_common();
|
|
97907
97756
|
init_recognition_util();
|
|
97757
|
+
init_warn();
|
|
97908
97758
|
|
|
97909
97759
|
// src/workflows/source-files.ts
|
|
97910
97760
|
init_common();
|
|
97911
97761
|
init_errors();
|
|
97912
97762
|
init_recognition_util();
|
|
97913
97763
|
init_warn();
|
|
97914
|
-
import fs49 from "fs";
|
|
97915
|
-
import path60 from "path";
|
|
97916
97764
|
|
|
97917
|
-
|
|
97918
|
-
|
|
97919
|
-
|
|
97920
|
-
super(message, code);
|
|
97921
|
-
this.sourcePaths = [...sourcePaths].sort(compareCodePoints);
|
|
97922
|
-
Object.setPrototypeOf(this, new.target.prototype);
|
|
97923
|
-
}
|
|
97924
|
-
}
|
|
97925
|
-
class WorkflowSourceLinkIdentityError extends WorkflowSourceRejectionError {
|
|
97926
|
-
constructor(sourcePath, targetPath) {
|
|
97927
|
-
super(`Workflow source ${sourcePath} resolves through a symlink to ${targetPath} with a different source format. ` + "The authored workflow path and resolved source must use the same .md or .yml format.", "WORKFLOW_SOURCE_INVALID", [sourcePath]);
|
|
97928
|
-
this.name = "WorkflowSourceLinkIdentityError";
|
|
97929
|
-
Object.setPrototypeOf(this, new.target.prototype);
|
|
97930
|
-
}
|
|
97931
|
-
}
|
|
97765
|
+
// src/indexer/enrich.ts
|
|
97766
|
+
init_errors();
|
|
97767
|
+
init_warn();
|
|
97932
97768
|
|
|
97933
|
-
|
|
97934
|
-
|
|
97935
|
-
|
|
97936
|
-
|
|
97937
|
-
|
|
97938
|
-
this.name = "WorkflowSourceLinkResolutionError";
|
|
97939
|
-
Object.setPrototypeOf(this, new.target.prototype);
|
|
97940
|
-
}
|
|
97941
|
-
}
|
|
97769
|
+
// src/llm/index-passes.ts
|
|
97770
|
+
init_warn();
|
|
97771
|
+
var NO_LOWERING_NOTICES = Object.freeze([]);
|
|
97772
|
+
// src/llm/structured-call.ts
|
|
97773
|
+
init_errors();
|
|
97942
97774
|
|
|
97943
|
-
|
|
97944
|
-
|
|
97945
|
-
|
|
97946
|
-
|
|
97947
|
-
|
|
97948
|
-
|
|
97949
|
-
|
|
97950
|
-
|
|
97775
|
+
// src/indexer/enrich.ts
|
|
97776
|
+
init_metadata();
|
|
97777
|
+
|
|
97778
|
+
// src/indexer/units/unit.ts
|
|
97779
|
+
init_markdown();
|
|
97780
|
+
|
|
97781
|
+
// src/core/hash.ts
|
|
97782
|
+
import { createHash as createHash10 } from "crypto";
|
|
97783
|
+
function hashEmbeddableText(text) {
|
|
97784
|
+
return createHash10("sha256").update(text, "utf8").digest("hex");
|
|
97951
97785
|
}
|
|
97952
|
-
|
|
97953
|
-
|
|
97954
|
-
|
|
97955
|
-
|
|
97956
|
-
|
|
97957
|
-
|
|
97958
|
-
|
|
97959
|
-
|
|
97960
|
-
|
|
97961
|
-
const extension = path60.posix.extname(ownedPath);
|
|
97962
|
-
if (!WORKFLOW_EXTENSIONS.includes(extension.toLowerCase()))
|
|
97963
|
-
return;
|
|
97964
|
-
return ownedPath;
|
|
97786
|
+
|
|
97787
|
+
// src/indexer/units/unit.ts
|
|
97788
|
+
init_metadata();
|
|
97789
|
+
var UNIT_HEADER_SECTION_SEPARATOR = " \u203A ";
|
|
97790
|
+
function structuredFieldsText(source) {
|
|
97791
|
+
const body = [source.description, source.tags, source.hints, source.parameters].filter((field) => field.length > 0).join(`
|
|
97792
|
+
`);
|
|
97793
|
+
return `${source.name}
|
|
97794
|
+
${body}`;
|
|
97965
97795
|
}
|
|
97966
|
-
function
|
|
97967
|
-
if (
|
|
97968
|
-
return
|
|
97969
|
-
|
|
97970
|
-
|
|
97971
|
-
|
|
97972
|
-
|
|
97973
|
-
}
|
|
97974
|
-
|
|
97975
|
-
|
|
97976
|
-
const
|
|
97977
|
-
|
|
97978
|
-
|
|
97979
|
-
|
|
97980
|
-
|
|
97981
|
-
|
|
97982
|
-
|
|
97983
|
-
|
|
97984
|
-
|
|
97985
|
-
|
|
97986
|
-
const canonicalName = canonicalizeWorkflowName(authoredName);
|
|
97987
|
-
if (!isSafeRelativeName(canonicalName))
|
|
97988
|
-
continue;
|
|
97989
|
-
const extension = path60.extname(normalizedSourcePath);
|
|
97990
|
-
const lowerExtension = extension.toLowerCase();
|
|
97991
|
-
if (!WORKFLOW_EXTENSIONS.includes(lowerExtension))
|
|
97992
|
-
continue;
|
|
97993
|
-
const candidate = {
|
|
97994
|
-
path: normalizedSourcePath,
|
|
97995
|
-
relativePath: toPosix(path60.relative(authoredRoot, normalizedSourcePath)),
|
|
97996
|
-
lowerExtension,
|
|
97997
|
-
extensionlessStem: path60.basename(normalizedSourcePath).slice(0, -extension.length)
|
|
97998
|
-
};
|
|
97999
|
-
const domain = candidatesByName.get(canonicalName) ?? [];
|
|
98000
|
-
domain.push(candidate);
|
|
98001
|
-
candidatesByName.set(canonicalName, domain);
|
|
98002
|
-
}
|
|
98003
|
-
const resolutions = [];
|
|
98004
|
-
for (const canonicalName of [...candidatesByName.keys()].sort(compareCodePoints)) {
|
|
98005
|
-
const candidates = candidatesByName.get(canonicalName) ?? [];
|
|
98006
|
-
candidates.sort((left, right) => compareCodePoints(left.relativePath, right.relativePath));
|
|
98007
|
-
const sources = inspectWorkflowSourceDomain(candidates, canonicalName, realRoot);
|
|
98008
|
-
const sourcePaths2 = candidates.map((candidate) => candidate.relativePath);
|
|
98009
|
-
resolutions.push({
|
|
98010
|
-
canonicalName,
|
|
98011
|
-
sourcePaths: sourcePaths2,
|
|
98012
|
-
source: pickWorkflowSource(adapterId, canonicalName, sources)
|
|
98013
|
-
});
|
|
98014
|
-
}
|
|
98015
|
-
return resolutions;
|
|
97796
|
+
function parametersText(parameters) {
|
|
97797
|
+
if (!parameters || parameters.length === 0)
|
|
97798
|
+
return "";
|
|
97799
|
+
return parameters.map((param) => param.description ? `${param.name}: ${param.description}` : param.name).join(`
|
|
97800
|
+
`).toLowerCase();
|
|
97801
|
+
}
|
|
97802
|
+
function fragmentHeaderText(name, sectionTitle) {
|
|
97803
|
+
return sectionTitle ? `${name}${UNIT_HEADER_SECTION_SEPARATOR}${sectionTitle}` : name;
|
|
97804
|
+
}
|
|
97805
|
+
function fragmentSectionTitles(safeMarkdown, fragments) {
|
|
97806
|
+
const headings = parseMarkdownToc(safeMarkdown).headings;
|
|
97807
|
+
let headingIndex = 0;
|
|
97808
|
+
let current = null;
|
|
97809
|
+
return fragments.map((fragment) => {
|
|
97810
|
+
while (headingIndex < headings.length && headings[headingIndex].line <= fragment.startLine) {
|
|
97811
|
+
current = headings[headingIndex].text;
|
|
97812
|
+
headingIndex++;
|
|
97813
|
+
}
|
|
97814
|
+
return current;
|
|
97815
|
+
});
|
|
98016
97816
|
}
|
|
98017
|
-
function
|
|
98018
|
-
const
|
|
98019
|
-
|
|
98020
|
-
|
|
98021
|
-
|
|
98022
|
-
|
|
97817
|
+
function splitOverflowingText(text, maxChars) {
|
|
97818
|
+
const pieces = [];
|
|
97819
|
+
let rest = text;
|
|
97820
|
+
while (rest.length > maxChars) {
|
|
97821
|
+
const window2 = rest.slice(0, maxChars);
|
|
97822
|
+
const newlineCut = window2.lastIndexOf(`
|
|
97823
|
+
`);
|
|
97824
|
+
if (newlineCut > 0) {
|
|
97825
|
+
pieces.push(rest.slice(0, newlineCut));
|
|
97826
|
+
rest = rest.slice(newlineCut + 1);
|
|
98023
97827
|
continue;
|
|
98024
97828
|
}
|
|
98025
|
-
|
|
98026
|
-
|
|
97829
|
+
const spaceCut = window2.lastIndexOf(" ");
|
|
97830
|
+
if (spaceCut > 0) {
|
|
97831
|
+
pieces.push(rest.slice(0, spaceCut));
|
|
97832
|
+
rest = rest.slice(spaceCut + 1);
|
|
97833
|
+
continue;
|
|
98027
97834
|
}
|
|
97835
|
+
pieces.push(rest.slice(0, maxChars));
|
|
97836
|
+
rest = rest.slice(maxChars);
|
|
97837
|
+
}
|
|
97838
|
+
if (rest.length > 0)
|
|
97839
|
+
pieces.push(rest);
|
|
97840
|
+
return pieces;
|
|
97841
|
+
}
|
|
97842
|
+
function deriveUnits(source, maxChars) {
|
|
97843
|
+
if (!Number.isFinite(maxChars) || maxChars <= 0) {
|
|
97844
|
+
throw new RangeError("deriveUnits: maxChars must be a positive finite number");
|
|
97845
|
+
}
|
|
97846
|
+
const units = [];
|
|
97847
|
+
let ordinal = 0;
|
|
97848
|
+
const pushUnit = (fragmentId, text) => {
|
|
97849
|
+
units.push({ entryId: source.entryId, ordinal: ordinal++, fragmentId, hash: hashEmbeddableText(text), text });
|
|
97850
|
+
};
|
|
97851
|
+
for (const text of splitOverflowingText(structuredFieldsText(source), maxChars))
|
|
97852
|
+
pushUnit(null, text);
|
|
97853
|
+
if (source.safeMarkdown != null) {
|
|
97854
|
+
const fragments = splitMarkdownFragments(source.safeMarkdown);
|
|
97855
|
+
const sectionTitles = fragmentSectionTitles(source.safeMarkdown, fragments);
|
|
97856
|
+
fragments.forEach((fragment, index) => {
|
|
97857
|
+
const header = fragmentHeaderText(source.name, sectionTitles[index] ?? null);
|
|
97858
|
+
const text = `${header}
|
|
97859
|
+
${fragment.text}`;
|
|
97860
|
+
for (const piece of splitOverflowingText(text, maxChars))
|
|
97861
|
+
pushUnit(fragment.fragmentId, piece);
|
|
97862
|
+
});
|
|
98028
97863
|
}
|
|
98029
|
-
return
|
|
98030
|
-
}
|
|
98031
|
-
function pickWorkflowSource(adapterId, canonicalName, sources) {
|
|
98032
|
-
if (sources.length <= 1)
|
|
98033
|
-
return sources[0];
|
|
98034
|
-
const winner = [...sources].sort((left, right) => left.format === right.format ? 0 : left.format === "markdown" ? -1 : 1)[0];
|
|
98035
|
-
const shadowed = sources.filter((source) => source !== winner);
|
|
98036
|
-
const displayName = adapterId === "akm" ? `workflows/${canonicalName}` : canonicalName;
|
|
98037
|
-
warnOnce(`workflow-source-collision:${displayName}`, `Workflow "${displayName}" has both a .md and .yml source (${shadowed.map((source) => source.relativePath).join(", ")} shadowed by ${winner?.relativePath}); using the .md source. Remove the shadowed sibling to ` + "silence this warning.");
|
|
98038
|
-
return winner;
|
|
97864
|
+
return units;
|
|
98039
97865
|
}
|
|
98040
|
-
function
|
|
98041
|
-
const
|
|
98042
|
-
|
|
98043
|
-
try {
|
|
98044
|
-
authoredStat = fs49.lstatSync(candidate.path);
|
|
98045
|
-
} catch {
|
|
98046
|
-
issues.push(new WorkflowSourceLinkResolutionError(candidate.relativePath));
|
|
98047
|
-
return { issues };
|
|
98048
|
-
}
|
|
98049
|
-
const isLink = authoredStat.isSymbolicLink();
|
|
98050
|
-
if (!isLink && !authoredStat.isFile()) {
|
|
98051
|
-
issues.push(new WorkflowSourceLinkResolutionError(candidate.relativePath));
|
|
98052
|
-
return { issues };
|
|
98053
|
-
}
|
|
98054
|
-
let realPath;
|
|
98055
|
-
try {
|
|
98056
|
-
realPath = fs49.realpathSync(candidate.path);
|
|
98057
|
-
} catch {
|
|
98058
|
-
issues.push(new WorkflowSourceLinkResolutionError(candidate.relativePath));
|
|
98059
|
-
return { issues };
|
|
98060
|
-
}
|
|
98061
|
-
const targetPath = toPosix(path60.relative(realRoot, realPath));
|
|
98062
|
-
const contained2 = isWithinResolved(realPath, realRoot);
|
|
98063
|
-
if (!contained2)
|
|
98064
|
-
issues.push(new WorkflowSourcePathIdentityError(candidate.relativePath, targetPath));
|
|
98065
|
-
if (isLink && path60.extname(realPath).toLowerCase() !== candidate.lowerExtension) {
|
|
98066
|
-
issues.push(new WorkflowSourceLinkIdentityError(candidate.relativePath, targetPath));
|
|
98067
|
-
}
|
|
98068
|
-
if (contained2) {
|
|
98069
|
-
try {
|
|
98070
|
-
if (!fs49.statSync(realPath).isFile())
|
|
98071
|
-
issues.push(new WorkflowSourceLinkResolutionError(candidate.relativePath));
|
|
98072
|
-
} catch {
|
|
98073
|
-
issues.push(new WorkflowSourceLinkResolutionError(candidate.relativePath));
|
|
98074
|
-
}
|
|
98075
|
-
}
|
|
98076
|
-
if (issues.length > 0)
|
|
98077
|
-
return { issues };
|
|
97866
|
+
function toUnitSource(entryId, entry) {
|
|
97867
|
+
const fields = buildSearchFields(entry);
|
|
97868
|
+
const safeMarkdown = hasMarkdownFragmentContent(entry) ? getMarkdownFragmentContent(entry) ?? null : typeof entry.content === "string" && entry.content.trim() ? entry.content : null;
|
|
98078
97869
|
return {
|
|
98079
|
-
|
|
98080
|
-
|
|
98081
|
-
|
|
98082
|
-
|
|
98083
|
-
|
|
98084
|
-
|
|
98085
|
-
|
|
98086
|
-
}
|
|
97870
|
+
entryId,
|
|
97871
|
+
name: fields.name,
|
|
97872
|
+
description: fields.description,
|
|
97873
|
+
tags: fields.tags,
|
|
97874
|
+
hints: fields.hints,
|
|
97875
|
+
parameters: parametersText(entry.parameters),
|
|
97876
|
+
safeMarkdown
|
|
98087
97877
|
};
|
|
98088
97878
|
}
|
|
98089
|
-
function isSafeRelativeName(name) {
|
|
98090
|
-
return name.length > 0 && !path60.posix.isAbsolute(name) && name !== ".." && !name.startsWith("../") && path60.posix.normalize(name) === name;
|
|
98091
|
-
}
|
|
98092
|
-
function isWithinResolved(candidate, root2) {
|
|
98093
|
-
const relative = path60.relative(root2, path60.resolve(candidate));
|
|
98094
|
-
return relative === "" || !relative.startsWith("..") && !path60.isAbsolute(relative);
|
|
98095
|
-
}
|
|
98096
97879
|
|
|
98097
|
-
// src/indexer/
|
|
97880
|
+
// src/indexer/reconcile.ts
|
|
97881
|
+
init_metadata();
|
|
97882
|
+
|
|
97883
|
+
// src/indexer/scan/parse-file.ts
|
|
98098
97884
|
init_compile();
|
|
98099
97885
|
init_metadata();
|
|
98100
|
-
init_file_context();
|
|
98101
97886
|
|
|
98102
97887
|
// src/indexer/scan/doc-to-entry.ts
|
|
98103
97888
|
init_metadata();
|
|
98104
|
-
import
|
|
97889
|
+
import path60 from "path";
|
|
98105
97890
|
function indexDocumentToStashEntry(doc) {
|
|
98106
97891
|
const dj = doc.documentJson ?? {};
|
|
98107
97892
|
const entry = {
|
|
98108
97893
|
name: doc.name,
|
|
98109
97894
|
type: doc.type,
|
|
98110
|
-
filename:
|
|
97895
|
+
filename: path60.basename(doc.path ?? "")
|
|
98111
97896
|
};
|
|
98112
97897
|
if (doc.description !== undefined)
|
|
98113
97898
|
entry.description = doc.description;
|
|
@@ -98203,58 +97988,25 @@ function isIntent(value) {
|
|
|
98203
97988
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
98204
97989
|
}
|
|
98205
97990
|
|
|
98206
|
-
// src/indexer/scan/
|
|
97991
|
+
// src/indexer/scan/parse-file.ts
|
|
98207
97992
|
var WORKFLOW_MD_RENDERER = "workflow-md";
|
|
98208
|
-
function
|
|
98209
|
-
const
|
|
98210
|
-
|
|
98211
|
-
|
|
98212
|
-
|
|
98213
|
-
|
|
98214
|
-
|
|
98215
|
-
|
|
98216
|
-
|
|
98217
|
-
|
|
98218
|
-
const leftName = workflowNameForSourcePath(component.root, adapter.id, left.absPath);
|
|
98219
|
-
const rightName = workflowNameForSourcePath(component.root, adapter.id, right.absPath);
|
|
98220
|
-
const leftOwner = leftName !== undefined && workflowOwnerPathByCanonicalName.get(canonicalizeWorkflowName(leftName)) === path63.resolve(left.absPath);
|
|
98221
|
-
const rightOwner = rightName !== undefined && workflowOwnerPathByCanonicalName.get(canonicalizeWorkflowName(rightName)) === path63.resolve(right.absPath);
|
|
98222
|
-
if (leftOwner !== rightOwner)
|
|
98223
|
-
return leftOwner ? -1 : 1;
|
|
98224
|
-
return compareCodePoints(left.absPath, right.absPath);
|
|
98225
|
-
});
|
|
98226
|
-
for (const file of orderedFileContexts) {
|
|
98227
|
-
if (rejectedPaths.has(file.absPath))
|
|
98228
|
-
continue;
|
|
98229
|
-
const workflowName = workflowNameForSourcePath(component.root, adapter.id, file.absPath);
|
|
98230
|
-
if (workflowName !== undefined) {
|
|
98231
|
-
const canonicalName = canonicalizeWorkflowName(workflowName);
|
|
98232
|
-
const ownerPath = workflowOwnerPathByCanonicalName.get(canonicalName);
|
|
98233
|
-
if (ownerPath !== undefined && ownerPath !== path63.resolve(file.absPath) && !invalidWorkflowOwnerNames.has(canonicalName)) {
|
|
98234
|
-
continue;
|
|
98235
|
-
}
|
|
98236
|
-
}
|
|
98237
|
-
const doc = adapter.recognize(component, file);
|
|
98238
|
-
if (doc === null)
|
|
98239
|
-
continue;
|
|
98240
|
-
if (!doc.conceptId) {
|
|
98241
|
-
warnings.push(`Skipped ${file.absPath}: adapter "${adapter.id}" returned no conceptId.`);
|
|
98242
|
-
continue;
|
|
98243
|
-
}
|
|
98244
|
-
const entry = indexDocumentToStashEntry(doc);
|
|
98245
|
-
const dropWarning = handleWorkflowDoc(doc, file, component.root);
|
|
98246
|
-
if (dropWarning !== null) {
|
|
98247
|
-
warnings.push(dropWarning);
|
|
98248
|
-
if (workflowName !== undefined)
|
|
98249
|
-
invalidWorkflowOwnerNames.add(canonicalizeWorkflowName(workflowName));
|
|
98250
|
-
continue;
|
|
98251
|
-
}
|
|
98252
|
-
if (doc.hash !== undefined)
|
|
98253
|
-
hashByFile.set(file.absPath, doc.hash);
|
|
98254
|
-
conceptIdByFile.set(file.absPath, doc.conceptId);
|
|
98255
|
-
entries.push(entry);
|
|
97993
|
+
function parseFileDocument(adapter, component, file) {
|
|
97994
|
+
const doc = adapter.recognize(component, file);
|
|
97995
|
+
if (doc === null)
|
|
97996
|
+
return { parsed: null, warning: null, isWorkflowDrop: false };
|
|
97997
|
+
if (!doc.conceptId) {
|
|
97998
|
+
return {
|
|
97999
|
+
parsed: null,
|
|
98000
|
+
warning: `Skipped ${file.absPath}: adapter "${adapter.id}" returned no conceptId.`,
|
|
98001
|
+
isWorkflowDrop: false
|
|
98002
|
+
};
|
|
98256
98003
|
}
|
|
98257
|
-
|
|
98004
|
+
const entry = indexDocumentToStashEntry(doc);
|
|
98005
|
+
const dropWarning = handleWorkflowDoc(doc, file, component.root);
|
|
98006
|
+
if (dropWarning !== null) {
|
|
98007
|
+
return { parsed: null, warning: dropWarning, isWorkflowDrop: true };
|
|
98008
|
+
}
|
|
98009
|
+
return { parsed: { entry, hash: doc.hash, conceptId: doc.conceptId }, warning: null, isWorkflowDrop: false };
|
|
98258
98010
|
}
|
|
98259
98011
|
function handleWorkflowDoc(doc, file, workspaceRoot) {
|
|
98260
98012
|
if (doc.type !== "workflow" || doc.adapterId !== "akm" && doc.adapterId !== "akm-workflow" || docRenderer(doc) !== WORKFLOW_MD_RENDERER && doc.adapterId !== "akm-workflow") {
|
|
@@ -98280,114 +98032,265 @@ function workflowDropWarning(file, errors3) {
|
|
|
98280
98032
|
${summary}`);
|
|
98281
98033
|
}
|
|
98282
98034
|
|
|
98283
|
-
// src/indexer/
|
|
98035
|
+
// src/indexer/reconcile.ts
|
|
98284
98036
|
init_file_context();
|
|
98037
|
+
|
|
98038
|
+
// src/indexer/walk/walker.ts
|
|
98039
|
+
init_asset_placement();
|
|
98040
|
+
init_recognition_util();
|
|
98041
|
+
init_file_context();
|
|
98042
|
+
var ALWAYS_SKIP_DIRS = new Set([".git"]);
|
|
98043
|
+
var AKM_SKIP_DIRS = new Set(["node_modules", "bin", ".cache"]);
|
|
98044
|
+
|
|
98045
|
+
// src/indexer/reconcile.ts
|
|
98046
|
+
async function reconcilePaths(db, paths, bundleId, opts) {
|
|
98047
|
+
const counts = emptyCounts();
|
|
98048
|
+
if (paths.length === 0)
|
|
98049
|
+
return counts;
|
|
98050
|
+
const config = loadConfig();
|
|
98051
|
+
let rootPath;
|
|
98052
|
+
let ctx;
|
|
98053
|
+
if (opts?.root) {
|
|
98054
|
+
const resolvedCtx = resolveRootContext(opts.root, bundleId);
|
|
98055
|
+
if (!resolvedCtx)
|
|
98056
|
+
return counts;
|
|
98057
|
+
rootPath = opts.root;
|
|
98058
|
+
ctx = resolvedCtx;
|
|
98059
|
+
} else {
|
|
98060
|
+
const resolvedRoot = resolveBundleRoot(bundleId, config);
|
|
98061
|
+
if (!resolvedRoot)
|
|
98062
|
+
return counts;
|
|
98063
|
+
rootPath = resolvedRoot.rootPath;
|
|
98064
|
+
ctx = { bundleId, component: resolvedRoot.component, adapter: resolvedRoot.adapter };
|
|
98065
|
+
}
|
|
98066
|
+
const maxChars = unitMaxChars(await probeProviderLimits(config.embedding ?? {}));
|
|
98067
|
+
for (const rawPath of paths) {
|
|
98068
|
+
counts.scanned++;
|
|
98069
|
+
const absPath = path62.resolve(rawPath);
|
|
98070
|
+
if (!fs51.existsSync(absPath)) {
|
|
98071
|
+
if (deleteFileAndEntryByPath(db, absPath))
|
|
98072
|
+
counts.removed++;
|
|
98073
|
+
continue;
|
|
98074
|
+
}
|
|
98075
|
+
const file = buildFileContext(rootPath, absPath);
|
|
98076
|
+
const classified = classifyFile(ctx, file, getFileState(db, absPath), (message) => counts.warnings.push(message));
|
|
98077
|
+
if (classified === "unchanged")
|
|
98078
|
+
counts.unchanged++;
|
|
98079
|
+
else if (classified === "unindexable") {
|
|
98080
|
+
if (deleteFileAndEntryByPath(db, absPath))
|
|
98081
|
+
counts.removed++;
|
|
98082
|
+
} else {
|
|
98083
|
+
applyOutcome(counts, applyChange(db, ctx, classified, maxChars, undefined, true));
|
|
98084
|
+
}
|
|
98085
|
+
}
|
|
98086
|
+
return counts;
|
|
98087
|
+
}
|
|
98088
|
+
function resolveRootContext(rootPath, bundleId) {
|
|
98089
|
+
const component = deriveInstallations([
|
|
98090
|
+
{ path: rootPath, registryId: bundleId, writable: true, adapterId: configuredAdapterIdForBundle(bundleId) }
|
|
98091
|
+
])[0]?.components[0];
|
|
98092
|
+
if (!component)
|
|
98093
|
+
return;
|
|
98094
|
+
const adapter = adapterForId(component.adapter);
|
|
98095
|
+
if (!adapter)
|
|
98096
|
+
return;
|
|
98097
|
+
return { bundleId, component, adapter };
|
|
98098
|
+
}
|
|
98099
|
+
function configuredAdapterIdForBundle(bundleId) {
|
|
98100
|
+
const bundle = loadConfig().bundles?.[bundleId];
|
|
98101
|
+
if (!bundle)
|
|
98102
|
+
return;
|
|
98103
|
+
return Object.values(bundle.components ?? {})[0]?.adapter;
|
|
98104
|
+
}
|
|
98105
|
+
function resolveBundleRoot(bundleId, config) {
|
|
98106
|
+
const sources = resolveSourceEntries(undefined, config);
|
|
98107
|
+
const installations = deriveInstallations(sources);
|
|
98108
|
+
const index = installations.findIndex((installation) => installation.id === bundleId);
|
|
98109
|
+
const source = index === -1 ? undefined : sources[index];
|
|
98110
|
+
const component = index === -1 ? undefined : installations[index]?.components[0];
|
|
98111
|
+
if (!source || !component)
|
|
98112
|
+
return;
|
|
98113
|
+
const adapter = adapterForId(component.adapter);
|
|
98114
|
+
if (!adapter)
|
|
98115
|
+
return;
|
|
98116
|
+
return { rootPath: source.path, component, adapter };
|
|
98117
|
+
}
|
|
98118
|
+
function classifyFile(ctx, file, storedHint, onWarning) {
|
|
98119
|
+
let stat;
|
|
98120
|
+
try {
|
|
98121
|
+
stat = file.stat();
|
|
98122
|
+
} catch {
|
|
98123
|
+
const access = classifyPathAccess(file.absPath);
|
|
98124
|
+
if (access.access === "inaccessible") {
|
|
98125
|
+
if (!storedHint) {
|
|
98126
|
+
onWarning?.(`New file akm cannot read, never indexed: ${describeInaccessiblePath(file.absPath, access.code)}`, false);
|
|
98127
|
+
}
|
|
98128
|
+
return "unchanged";
|
|
98129
|
+
}
|
|
98130
|
+
return "unindexable";
|
|
98131
|
+
}
|
|
98132
|
+
if (storedHint && storedHint.size === stat.size && storedHint.mtimeMs === stat.mtimeMs && storedHint.ctimeMs === stat.ctimeMs && storedHint.adapterId === ctx.adapter.id) {
|
|
98133
|
+
return "unchanged";
|
|
98134
|
+
}
|
|
98135
|
+
const outcome = parseFileDocument(ctx.adapter, ctx.component, file);
|
|
98136
|
+
if (outcome.parsed === null) {
|
|
98137
|
+
if (outcome.warning !== null)
|
|
98138
|
+
onWarning?.(outcome.warning, outcome.isWorkflowDrop);
|
|
98139
|
+
return "unindexable";
|
|
98140
|
+
}
|
|
98141
|
+
const hash4 = outcome.parsed.hash ?? hashContent(file.content());
|
|
98142
|
+
return { file, stat, entry: outcome.parsed.entry, conceptId: outcome.parsed.conceptId, hash: hash4 };
|
|
98143
|
+
}
|
|
98144
|
+
function applyChange(db, ctx, change, maxChars, renameSource, supersedeOtherBundles) {
|
|
98145
|
+
const { file, stat, entry, conceptId, hash: hash4 } = change;
|
|
98146
|
+
const searchText = buildSearchText(entry);
|
|
98147
|
+
const provenance = deriveEntryProvenance({ bundleId: ctx.bundleId, componentId: ctx.component.id, adapterId: ctx.component.adapter }, entry.type, entry.name, conceptId);
|
|
98148
|
+
const entryWithSize = { ...entry, fileSize: stat.size };
|
|
98149
|
+
if (hasMarkdownFragmentContent(entry))
|
|
98150
|
+
setMarkdownFragmentContent(entryWithSize, getMarkdownFragmentContent(entry));
|
|
98151
|
+
let orphanedUnitHashes = [];
|
|
98152
|
+
const result = withImmediateTransaction(db, () => {
|
|
98153
|
+
if (supersedeOtherBundles)
|
|
98154
|
+
supersedeOtherItemRefsAtPath(db, file.absPath, provenance.itemRef);
|
|
98155
|
+
const written = renameSource ? repointOrInsert(db, renameSource.path, file.absPath, entryWithSize, searchText, provenance, hash4) : upsertOrInsert(db, file.absPath, entryWithSize, searchText, provenance, hash4);
|
|
98156
|
+
if (renameSource)
|
|
98157
|
+
deleteFileStates(db, [renameSource.path]);
|
|
98158
|
+
upsertFileState(db, {
|
|
98159
|
+
path: file.absPath,
|
|
98160
|
+
bundleId: ctx.bundleId,
|
|
98161
|
+
size: stat.size,
|
|
98162
|
+
mtimeMs: stat.mtimeMs,
|
|
98163
|
+
ctimeMs: stat.ctimeMs,
|
|
98164
|
+
blobHash: hash4,
|
|
98165
|
+
adapterId: ctx.adapter.id
|
|
98166
|
+
});
|
|
98167
|
+
const units = deriveUnits(toUnitSource(written.entryId, entry), maxChars);
|
|
98168
|
+
const { inserted } = insertNewUnitTexts(db, units.map((unit) => ({
|
|
98169
|
+
hash: unit.hash,
|
|
98170
|
+
kind: unit.fragmentId === null ? "card" : "fragment",
|
|
98171
|
+
text: unit.text
|
|
98172
|
+
})));
|
|
98173
|
+
const previousHashes = db.prepare("SELECT unit_hash FROM entry_units WHERE entry_id = ?").all(written.entryId).map((row) => row.unit_hash);
|
|
98174
|
+
replaceEntryUnits(db, written.entryId, units.map((unit) => ({ ordinal: unit.ordinal, fragmentId: unit.fragmentId, hash: unit.hash })));
|
|
98175
|
+
const currentHashes = new Set(units.map((unit) => unit.hash));
|
|
98176
|
+
orphanedUnitHashes = previousHashes.filter((oldHash) => !currentHashes.has(oldHash));
|
|
98177
|
+
return {
|
|
98178
|
+
outcome: written.outcome,
|
|
98179
|
+
unitsAdded: inserted,
|
|
98180
|
+
entryId: written.entryId,
|
|
98181
|
+
entry: entryWithSize,
|
|
98182
|
+
provenance
|
|
98183
|
+
};
|
|
98184
|
+
}, "index");
|
|
98185
|
+
if (orphanedUnitHashes.length > 0)
|
|
98186
|
+
pruneOrphanUnitTextsForHashes(db, orphanedUnitHashes);
|
|
98187
|
+
return result;
|
|
98188
|
+
}
|
|
98189
|
+
function upsertOrInsert(db, filePath, entry, searchText, provenance, hash4) {
|
|
98190
|
+
const existedBefore = getFileState(db, filePath) !== undefined;
|
|
98191
|
+
const entryId = upsertEntry(db, filePath, entry, searchText, provenance, hash4);
|
|
98192
|
+
return { entryId, outcome: existedBefore ? "changed" : "added" };
|
|
98193
|
+
}
|
|
98194
|
+
function repointOrInsert(db, oldPath, newPath, entry, searchText, provenance, hash4) {
|
|
98195
|
+
const oldRow = db.prepare("SELECT id FROM entries WHERE file_path = ?").get(oldPath);
|
|
98196
|
+
if (!oldRow)
|
|
98197
|
+
return upsertOrInsert(db, newPath, entry, searchText, provenance, hash4);
|
|
98198
|
+
const claimedByOther = db.prepare("SELECT 1 FROM entries WHERE item_ref = ? AND id <> ?").get(provenance.itemRef, oldRow.id);
|
|
98199
|
+
if (claimedByOther)
|
|
98200
|
+
return upsertOrInsert(db, newPath, entry, searchText, provenance, hash4);
|
|
98201
|
+
repointEntry(db, oldRow.id, newPath, entry, searchText, provenance, hash4);
|
|
98202
|
+
return { entryId: oldRow.id, outcome: "changed" };
|
|
98203
|
+
}
|
|
98204
|
+
function repointEntry(db, entryId, filePath, entry, searchText, provenance, contentHash) {
|
|
98205
|
+
const derivedFrom = typeof entry.derivedFrom === "string" && entry.derivedFrom.trim() ? entry.derivedFrom.trim() : null;
|
|
98206
|
+
db.prepare(`UPDATE entries SET item_ref = ?, bundle_id = ?, component_id = ?, concept_id = ?, adapter_id = ?, type = ?,
|
|
98207
|
+
file_path = ?, content_hash = ?, document_json = ?, search_text = ?, derived_from = ?
|
|
98208
|
+
WHERE id = ?`).run(provenance.itemRef, provenance.bundleId, provenance.componentId, provenance.conceptId, provenance.adapterId, entry.type, filePath, contentHash, JSON.stringify(entry), searchText, derivedFrom, entryId);
|
|
98209
|
+
replaceFragmentSource(db, entryId, hasMarkdownFragmentContent(entry) ? getMarkdownFragmentContent(entry) ?? null : undefined);
|
|
98210
|
+
}
|
|
98211
|
+
function supersedeOtherItemRefsAtPath(db, filePath, keepItemRef) {
|
|
98212
|
+
const staleIds = db.prepare("SELECT id FROM entries WHERE file_path = ? AND item_ref <> ?").all(filePath, keepItemRef).map((row) => row.id);
|
|
98213
|
+
if (staleIds.length > 0)
|
|
98214
|
+
deleteEntriesByIds(db, staleIds);
|
|
98215
|
+
}
|
|
98216
|
+
function deleteFileAndEntryByPath(db, filePath) {
|
|
98217
|
+
return withImmediateTransaction(db, () => {
|
|
98218
|
+
const entryIds = db.prepare("SELECT id FROM entries WHERE file_path = ?").all(filePath).map((row) => row.id);
|
|
98219
|
+
if (entryIds.length > 0)
|
|
98220
|
+
deleteEntriesByIds(db, entryIds);
|
|
98221
|
+
const hadFileRow = getFileState(db, filePath) !== undefined;
|
|
98222
|
+
if (hadFileRow)
|
|
98223
|
+
deleteFileStates(db, [filePath]);
|
|
98224
|
+
return entryIds.length > 0 || hadFileRow;
|
|
98225
|
+
}, "index");
|
|
98226
|
+
}
|
|
98227
|
+
function emptyCounts() {
|
|
98228
|
+
return { scanned: 0, unchanged: 0, added: 0, changed: 0, removed: 0, unitsAdded: 0, complete: true, warnings: [] };
|
|
98229
|
+
}
|
|
98230
|
+
function applyOutcome(counts, result) {
|
|
98231
|
+
if (result.outcome === "added")
|
|
98232
|
+
counts.added++;
|
|
98233
|
+
else
|
|
98234
|
+
counts.changed++;
|
|
98235
|
+
counts.unitsAdded += result.unitsAdded;
|
|
98236
|
+
}
|
|
98237
|
+
|
|
98238
|
+
// src/indexer/index-written-assets.ts
|
|
98285
98239
|
var WRITE_PATH_INDEX_BUSY_TIMEOUT_MS = 5000;
|
|
98240
|
+
function unitHashesForFiles(db, files) {
|
|
98241
|
+
const hashes = new Set;
|
|
98242
|
+
for (let offset = 0;offset < files.length; offset += SQLITE_CHUNK_SIZE) {
|
|
98243
|
+
const chunk2 = files.slice(offset, offset + SQLITE_CHUNK_SIZE);
|
|
98244
|
+
const placeholders = chunk2.map(() => "?").join(",");
|
|
98245
|
+
const rows = db.prepare(`SELECT DISTINCT eu.unit_hash AS unitHash
|
|
98246
|
+
FROM entries e
|
|
98247
|
+
JOIN entry_units eu ON eu.entry_id = e.id
|
|
98248
|
+
WHERE e.file_path IN (${placeholders})`).all(...chunk2);
|
|
98249
|
+
for (const row of rows)
|
|
98250
|
+
hashes.add(row.unitHash);
|
|
98251
|
+
}
|
|
98252
|
+
return [...hashes];
|
|
98253
|
+
}
|
|
98254
|
+
function resolveWrittenAssetBundleId(stashDir) {
|
|
98255
|
+
const resolvedStashDir = path63.resolve(stashDir);
|
|
98256
|
+
const sourceEntries = resolveSourceEntries();
|
|
98257
|
+
const index = sourceEntries.findIndex((entry) => path63.resolve(entry.path) === resolvedStashDir);
|
|
98258
|
+
if (index !== -1)
|
|
98259
|
+
return deriveInstallations(sourceEntries)[index]?.id;
|
|
98260
|
+
return deriveInstallations([{ path: stashDir, writable: true }])[0]?.id;
|
|
98261
|
+
}
|
|
98286
98262
|
async function indexWrittenAssets(stashDir, filePaths, options = {}) {
|
|
98287
98263
|
try {
|
|
98288
98264
|
return await (async () => {
|
|
98289
|
-
const rebuildProbe = probeLock(getIndexRebuildLockPath());
|
|
98290
|
-
if (rebuildProbe.state === "held") {
|
|
98291
|
-
const holderLabel = formatLockHolderPid({
|
|
98292
|
-
pid: rebuildProbe.holderPid,
|
|
98293
|
-
launcherPid: rebuildProbe.launcherPid ?? null
|
|
98294
|
-
});
|
|
98295
|
-
warn(`index rebuild in progress (pid ${holderLabel}); the next index pass will index ${filePaths.join(", ")}`);
|
|
98296
|
-
return true;
|
|
98297
|
-
}
|
|
98298
98265
|
const dbPath = getDbPath();
|
|
98299
98266
|
if (isPathAbsent(dbPath))
|
|
98300
98267
|
return true;
|
|
98301
98268
|
const files = filePaths.filter((f) => {
|
|
98302
|
-
const rel =
|
|
98269
|
+
const rel = path63.relative(stashDir, f);
|
|
98303
98270
|
return !rel.split(/[\\/]+/).some((segment) => segment.startsWith("."));
|
|
98304
98271
|
});
|
|
98305
98272
|
if (files.length === 0)
|
|
98306
98273
|
return true;
|
|
98307
|
-
const
|
|
98308
|
-
|
|
98309
|
-
])[0]?.components[0];
|
|
98310
|
-
if (!component)
|
|
98274
|
+
const bundleId = options.bundleId ?? resolveWrittenAssetBundleId(stashDir);
|
|
98275
|
+
if (!bundleId)
|
|
98311
98276
|
throw new Error(`Could not derive bundle provenance for ${stashDir}`);
|
|
98312
|
-
const pairs = [];
|
|
98313
|
-
const unindexable = new Set;
|
|
98314
|
-
const rejectedConceptIds = new Set;
|
|
98315
|
-
for (const file of files) {
|
|
98316
|
-
if (!fs53.existsSync(file)) {
|
|
98317
|
-
let authoredDanglingSymlink = false;
|
|
98318
|
-
try {
|
|
98319
|
-
authoredDanglingSymlink = fs53.lstatSync(file).isSymbolicLink();
|
|
98320
|
-
} catch {}
|
|
98321
|
-
if (!authoredDanglingSymlink) {
|
|
98322
|
-
unindexable.add(file);
|
|
98323
|
-
continue;
|
|
98324
|
-
}
|
|
98325
|
-
}
|
|
98326
|
-
const ctx = buildFileContext(stashDir, file);
|
|
98327
|
-
const drained = drainDirDocuments(akmAdapter, component, [ctx]);
|
|
98328
|
-
for (const rejectedPath of drained.rejectedPaths)
|
|
98329
|
-
unindexable.add(rejectedPath);
|
|
98330
|
-
for (const conceptId2 of drained.rejectedConceptIds)
|
|
98331
|
-
rejectedConceptIds.add(conceptId2);
|
|
98332
|
-
const entry = drained.entries[0];
|
|
98333
|
-
const conceptId = drained.conceptIdByFile.get(ctx.absPath);
|
|
98334
|
-
if (entry && conceptId)
|
|
98335
|
-
pairs.push({ file, entry, conceptId, contentHash: drained.hashByFile.get(ctx.absPath) });
|
|
98336
|
-
else
|
|
98337
|
-
unindexable.add(file);
|
|
98338
|
-
}
|
|
98339
98277
|
const db = openExistingDatabase(dbPath);
|
|
98340
98278
|
try {
|
|
98341
98279
|
db.exec(`PRAGMA busy_timeout = ${WRITE_PATH_INDEX_BUSY_TIMEOUT_MS}`);
|
|
98342
98280
|
if (getEntryCount(db) === 0)
|
|
98343
98281
|
return true;
|
|
98344
|
-
|
|
98345
|
-
|
|
98346
|
-
db.transaction(() => {
|
|
98347
|
-
const unindexableEntryIds = new Set;
|
|
98348
|
-
for (const file of unindexable) {
|
|
98349
|
-
const rows = db.prepare(`SELECT id FROM entries
|
|
98350
|
-
WHERE file_path = ?
|
|
98351
|
-
AND bundle_id = ?
|
|
98352
|
-
AND adapter_id = ?`).all(file, component.id, component.adapter);
|
|
98353
|
-
for (const row of rows)
|
|
98354
|
-
unindexableEntryIds.add(row.id);
|
|
98355
|
-
}
|
|
98356
|
-
for (const conceptId of rejectedConceptIds) {
|
|
98357
|
-
const itemRef = `${component.id}//${conceptId}`;
|
|
98358
|
-
const rows = db.prepare(`SELECT id FROM entries
|
|
98359
|
-
WHERE bundle_id = ? AND adapter_id = ?
|
|
98360
|
-
AND type = 'workflow'
|
|
98361
|
-
AND (concept_id = ? OR item_ref = ?)`).all(component.id, component.adapter, conceptId, itemRef);
|
|
98362
|
-
for (const row of rows)
|
|
98363
|
-
unindexableEntryIds.add(row.id);
|
|
98364
|
-
}
|
|
98365
|
-
deleteEntriesByIds(db, [...unindexableEntryIds]);
|
|
98366
|
-
mutated ||= unindexableEntryIds.size > 0;
|
|
98367
|
-
for (const { file, entry, conceptId, contentHash } of pairs) {
|
|
98368
|
-
let entryWithSize = entry;
|
|
98369
|
-
try {
|
|
98370
|
-
entryWithSize = { ...entry, fileSize: fs53.statSync(file).size };
|
|
98371
|
-
if (hasMarkdownFragmentContent(entry)) {
|
|
98372
|
-
setMarkdownFragmentContent(entryWithSize, getMarkdownFragmentContent(entry));
|
|
98373
|
-
}
|
|
98374
|
-
} catch {}
|
|
98375
|
-
const provenance = deriveEntryProvenance({ bundleId: component.id, componentId: component.id, adapterId: component.adapter }, entry.type, entry.name, conceptId);
|
|
98376
|
-
const supersededIds = db.prepare("SELECT id FROM entries WHERE file_path = ? AND item_ref <> ?").all(file, provenance.itemRef);
|
|
98377
|
-
deleteEntriesByIds(db, supersededIds.map((row) => row.id));
|
|
98378
|
-
targetEntryIds.add(upsertEntry(db, file, entryWithSize, buildSearchText(entry), provenance, contentHash));
|
|
98379
|
-
mutated = true;
|
|
98380
|
-
}
|
|
98381
|
-
})();
|
|
98382
|
-
if (mutated) {
|
|
98282
|
+
await reconcilePaths(db, files, bundleId, { root: stashDir });
|
|
98283
|
+
try {
|
|
98383
98284
|
const config = loadConfig();
|
|
98384
|
-
|
|
98385
|
-
|
|
98285
|
+
const onlyHashes = unitHashesForFiles(db, files);
|
|
98286
|
+
await drainEmbeddingQueue(db, config, { onlyHashes });
|
|
98287
|
+
} catch (drainError) {
|
|
98288
|
+
warnVerbose("Write-path embedding drain skipped (vectors appear after the next drain):", drainError instanceof Error ? drainError.message : String(drainError));
|
|
98386
98289
|
}
|
|
98290
|
+
return true;
|
|
98387
98291
|
} finally {
|
|
98388
98292
|
closeDatabase(db);
|
|
98389
98293
|
}
|
|
98390
|
-
return true;
|
|
98391
98294
|
})();
|
|
98392
98295
|
} catch (error2) {
|
|
98393
98296
|
if (isDataDirUnreadableError(error2)) {
|
|
@@ -98403,7 +98306,7 @@ async function indexWrittenAssets(stashDir, filePaths, options = {}) {
|
|
|
98403
98306
|
init_asset_placement();
|
|
98404
98307
|
init_asset_ref();
|
|
98405
98308
|
init_warn();
|
|
98406
|
-
import
|
|
98309
|
+
import path64 from "path";
|
|
98407
98310
|
function changesToStored(changes) {
|
|
98408
98311
|
return changes.map((c, i) => ({
|
|
98409
98312
|
path: c.path,
|
|
@@ -98456,10 +98359,10 @@ function currentProposalTarget(value) {
|
|
|
98456
98359
|
if (typeof value !== "object" || value === null)
|
|
98457
98360
|
throw new Error("Proposal metadata has an invalid proposedTarget.");
|
|
98458
98361
|
const target = value;
|
|
98459
|
-
if (typeof target.source !== "string" || !isBundleSlug(target.source) || typeof target.root !== "string" || !
|
|
98362
|
+
if (typeof target.source !== "string" || !isBundleSlug(target.source) || typeof target.root !== "string" || !path64.isAbsolute(target.root)) {
|
|
98460
98363
|
throw new Error("Proposal metadata has an invalid proposedTarget.");
|
|
98461
98364
|
}
|
|
98462
|
-
return { source: target.source, root:
|
|
98365
|
+
return { source: target.source, root: path64.resolve(target.root) };
|
|
98463
98366
|
}
|
|
98464
98367
|
function invalidPresentField(name) {
|
|
98465
98368
|
throw new Error(`Proposal metadata has an invalid ${name}.`);
|
|
@@ -98499,7 +98402,7 @@ function validatePresentMetadata(meta) {
|
|
|
98499
98402
|
}
|
|
98500
98403
|
if (Object.hasOwn(meta, "acceptedTarget")) {
|
|
98501
98404
|
const target = meta.acceptedTarget;
|
|
98502
|
-
if (typeof target !== "object" || target === null || typeof target.source !== "string" || !isBundleSlug(target.source) || typeof target.root !== "string" || !
|
|
98405
|
+
if (typeof target !== "object" || target === null || typeof target.source !== "string" || !isBundleSlug(target.source) || typeof target.root !== "string" || !path64.isAbsolute(target.root) || typeof target.path !== "string" || !path64.isAbsolute(target.path) || typeof target.contentHash !== "string") {
|
|
98503
98406
|
invalidPresentField("acceptedTarget");
|
|
98504
98407
|
}
|
|
98505
98408
|
}
|
|
@@ -99269,12 +99172,12 @@ function proposalHash(content) {
|
|
|
99269
99172
|
return createHash11("sha256").update(content).digest("hex");
|
|
99270
99173
|
}
|
|
99271
99174
|
function proposalFileHash(filePath) {
|
|
99272
|
-
return proposalHash(
|
|
99175
|
+
return proposalHash(fs53.readFileSync(filePath));
|
|
99273
99176
|
}
|
|
99274
99177
|
function sameProposalFile(left, right) {
|
|
99275
99178
|
try {
|
|
99276
|
-
const leftStat =
|
|
99277
|
-
const rightStat =
|
|
99179
|
+
const leftStat = fs53.statSync(left);
|
|
99180
|
+
const rightStat = fs53.statSync(right);
|
|
99278
99181
|
return leftStat.dev === rightStat.dev && leftStat.ino === rightStat.ino;
|
|
99279
99182
|
} catch {
|
|
99280
99183
|
return false;
|
|
@@ -99283,20 +99186,20 @@ function sameProposalFile(left, right) {
|
|
|
99283
99186
|
function cleanupProposalPublication(p) {
|
|
99284
99187
|
for (const filePath of [p.publishPath, p.displacedPath]) {
|
|
99285
99188
|
try {
|
|
99286
|
-
|
|
99189
|
+
fs53.rmSync(filePath, { force: true });
|
|
99287
99190
|
} catch (error2) {
|
|
99288
99191
|
warn(`[proposals] transaction publication cleanup failed at ${filePath}: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
99289
99192
|
}
|
|
99290
99193
|
}
|
|
99291
|
-
fsyncTxnDir(
|
|
99194
|
+
fsyncTxnDir(path65.dirname(p.assetPath));
|
|
99292
99195
|
}
|
|
99293
99196
|
function rollbackPreparedProposalTransaction(txn) {
|
|
99294
99197
|
const p = txn.journal.payload;
|
|
99295
|
-
const currentHash =
|
|
99296
|
-
if (!
|
|
99198
|
+
const currentHash = fs53.existsSync(p.assetPath) ? proposalFileHash(p.assetPath) : null;
|
|
99199
|
+
if (!fs53.existsSync(p.displacedPath)) {
|
|
99297
99200
|
if (p.originalHash === null) {
|
|
99298
99201
|
if (currentHash === p.publishedHash && sameProposalFile(p.assetPath, p.publishPath)) {
|
|
99299
|
-
|
|
99202
|
+
fs53.unlinkSync(p.assetPath);
|
|
99300
99203
|
recordWrittenPath(p.assetPath);
|
|
99301
99204
|
} else if (currentHash !== null) {
|
|
99302
99205
|
throw new Error(`Cannot roll back proposal transaction: target was created externally.`);
|
|
@@ -99308,30 +99211,30 @@ function rollbackPreparedProposalTransaction(txn) {
|
|
|
99308
99211
|
return;
|
|
99309
99212
|
}
|
|
99310
99213
|
if (currentHash === p.publishedHash) {
|
|
99311
|
-
|
|
99214
|
+
fs53.unlinkSync(p.assetPath);
|
|
99312
99215
|
recordWrittenPath(p.assetPath);
|
|
99313
99216
|
} else if (currentHash !== null && currentHash !== p.originalHash) {
|
|
99314
99217
|
throw new Error(`Cannot roll back proposal transaction: ${p.assetPath} diverged.`);
|
|
99315
99218
|
}
|
|
99316
|
-
if (
|
|
99317
|
-
if (
|
|
99219
|
+
if (fs53.existsSync(p.displacedPath)) {
|
|
99220
|
+
if (fs53.existsSync(p.assetPath)) {
|
|
99318
99221
|
throw new Error(`Cannot restore proposal backup: ${p.assetPath} is occupied.`);
|
|
99319
99222
|
}
|
|
99320
|
-
|
|
99223
|
+
fs53.linkSync(p.displacedPath, p.assetPath);
|
|
99321
99224
|
recordWrittenPath(p.assetPath);
|
|
99322
99225
|
}
|
|
99323
99226
|
cleanupProposalPublication(p);
|
|
99324
99227
|
}
|
|
99325
99228
|
function validatePublishedProposal(p) {
|
|
99326
|
-
if (!
|
|
99229
|
+
if (!fs53.existsSync(p.assetPath) || proposalFileHash(p.assetPath) !== p.publishedHash) {
|
|
99327
99230
|
throw new Error(`Cannot recover proposal ${p.proposalId}: published asset diverged.`);
|
|
99328
99231
|
}
|
|
99329
99232
|
}
|
|
99330
99233
|
function persistProposalTransactionState(txn, proposal, ctx) {
|
|
99331
99234
|
const p = txn.journal.payload;
|
|
99332
99235
|
const decidedAt = txn.journal.decidedAt;
|
|
99333
|
-
const backupContent = p.backupPath ?
|
|
99334
|
-
const publishedContent =
|
|
99236
|
+
const backupContent = p.backupPath ? fs53.readFileSync(p.backupPath, "utf8") : undefined;
|
|
99237
|
+
const publishedContent = fs53.readFileSync(p.contentPath, "utf8");
|
|
99335
99238
|
return withProposalsDb(p.stashDir, ctx, (db) => withImmediateTransaction(db, () => {
|
|
99336
99239
|
const current = requireProposal(db, p.stashDir, p.proposalId);
|
|
99337
99240
|
if (p.operation === "accept") {
|
|
@@ -99349,7 +99252,7 @@ function persistProposalTransactionState(txn, proposal, ctx) {
|
|
|
99349
99252
|
payload: { ...proposal.payload, content: publishedContent },
|
|
99350
99253
|
changes: [
|
|
99351
99254
|
{
|
|
99352
|
-
path:
|
|
99255
|
+
path: path65.relative(txn.journal.root, p.assetPath),
|
|
99353
99256
|
op: p.originalHash === null ? "create" : "update",
|
|
99354
99257
|
after: publishedContent
|
|
99355
99258
|
}
|
|
@@ -99420,7 +99323,7 @@ async function finalizeProposalTransaction(txn, target, proposal, ctx) {
|
|
|
99420
99323
|
cleanupProposalPublication(p);
|
|
99421
99324
|
if (txn.journal.phase === "asset-published") {
|
|
99422
99325
|
const commitRoot = target.source.repoPath ?? target.source.path;
|
|
99423
|
-
const commitPath =
|
|
99326
|
+
const commitPath = path65.relative(commitRoot, p.assetPath).replaceAll(path65.sep, "/");
|
|
99424
99327
|
publishWriteTargetTransaction(target, p.gitPublication, {
|
|
99425
99328
|
transactionId: txn.journal.transactionId,
|
|
99426
99329
|
message: `${p.operation === "accept" ? "Update" : "Revert"} ${p.ref}`,
|
|
@@ -99439,10 +99342,11 @@ async function finalizeProposalTransaction(txn, target, proposal, ctx) {
|
|
|
99439
99342
|
}
|
|
99440
99343
|
let accepted = getProposal(p.stashDir, p.proposalId, ctx);
|
|
99441
99344
|
if (txn.journal.phase === "proposal-persisted") {
|
|
99442
|
-
if (
|
|
99443
|
-
|
|
99345
|
+
if (await indexWrittenAssets(txn.journal.root, [p.assetPath], { bundleId: target.source.name })) {
|
|
99346
|
+
advanceTxn(txn, "index-finalized");
|
|
99347
|
+
} else {
|
|
99348
|
+
warn(`${p.operation === "accept" ? "Accept" : "Revert"} of ${p.ref} succeeded, but its index update failed; run \`akm index\` to refresh it.`);
|
|
99444
99349
|
}
|
|
99445
|
-
advanceTxn(txn, "index-finalized");
|
|
99446
99350
|
}
|
|
99447
99351
|
if (txn.journal.phase === "index-finalized") {
|
|
99448
99352
|
accepted = getProposal(p.stashDir, p.proposalId, ctx);
|
|
@@ -99457,8 +99361,8 @@ async function finalizeProposalTransaction(txn, target, proposal, ctx) {
|
|
|
99457
99361
|
function fenceProposalTxnJournal(journal, txnDir, root2) {
|
|
99458
99362
|
const p = journal.payload;
|
|
99459
99363
|
const refIdentity = proposalRefIdentity(p.ref);
|
|
99460
|
-
if (!["accept", "revert"].includes(p.operation) || !p.targetSource || !p.targetKind || refIdentity?.bundle === undefined || !isWithin(p.assetPath, root2) || ![p.contentPath, p.backupPath].filter((candidate) => candidate !== null).every((candidate) => isWithin(candidate, txnDir)) || ![p.publishPath, p.displacedPath].every((candidate) => isWithin(candidate, root2) &&
|
|
99461
|
-
throw new Error(`Refusing unsafe proposal transaction journal at ${
|
|
99364
|
+
if (!["accept", "revert"].includes(p.operation) || !p.targetSource || !p.targetKind || refIdentity?.bundle === undefined || !isWithin(p.assetPath, root2) || ![p.contentPath, p.backupPath].filter((candidate) => candidate !== null).every((candidate) => isWithin(candidate, txnDir)) || ![p.publishPath, p.displacedPath].every((candidate) => isWithin(candidate, root2) && path65.dirname(candidate) === path65.dirname(p.assetPath))) {
|
|
99365
|
+
throw new Error(`Refusing unsafe proposal transaction journal at ${path65.join(txnDir, "journal.json")}.`);
|
|
99462
99366
|
}
|
|
99463
99367
|
}
|
|
99464
99368
|
function resolveProposalRecoveryTarget(config, journal) {
|
|
@@ -99558,8 +99462,8 @@ async function recoverStaleTxns(stashDir) {
|
|
|
99558
99462
|
}
|
|
99559
99463
|
|
|
99560
99464
|
// scripts/akm-migrate/migrate/writer-relocation.ts
|
|
99561
|
-
import
|
|
99562
|
-
import
|
|
99465
|
+
import fs55 from "fs";
|
|
99466
|
+
import path66 from "path";
|
|
99563
99467
|
init_paths();
|
|
99564
99468
|
function relocationSpecs(stashDir) {
|
|
99565
99469
|
return [
|
|
@@ -99579,7 +99483,7 @@ function mutexSiblingName(lockName) {
|
|
|
99579
99483
|
function fileCountIfExists(dir) {
|
|
99580
99484
|
let entries;
|
|
99581
99485
|
try {
|
|
99582
|
-
entries =
|
|
99486
|
+
entries = fs55.readdirSync(dir, { withFileTypes: true });
|
|
99583
99487
|
} catch {
|
|
99584
99488
|
return;
|
|
99585
99489
|
}
|
|
@@ -99587,7 +99491,7 @@ function fileCountIfExists(dir) {
|
|
|
99587
99491
|
}
|
|
99588
99492
|
function statFileIfExists(filePath) {
|
|
99589
99493
|
try {
|
|
99590
|
-
const stat =
|
|
99494
|
+
const stat = fs55.statSync(filePath);
|
|
99591
99495
|
return stat.isFile() ? stat : undefined;
|
|
99592
99496
|
} catch {
|
|
99593
99497
|
return;
|
|
@@ -99597,8 +99501,8 @@ function classifyLockArtifacts(akmDir) {
|
|
|
99597
99501
|
const removable = [];
|
|
99598
99502
|
const skipped = [];
|
|
99599
99503
|
for (const lockName of LOCK_NAMES) {
|
|
99600
|
-
const lockPath =
|
|
99601
|
-
const mutexPath =
|
|
99504
|
+
const lockPath = path66.join(akmDir, lockName);
|
|
99505
|
+
const mutexPath = path66.join(akmDir, mutexSiblingName(lockName));
|
|
99602
99506
|
const lockStat = statFileIfExists(lockPath);
|
|
99603
99507
|
const mutexStat = statFileIfExists(mutexPath);
|
|
99604
99508
|
if (!lockStat) {
|
|
@@ -99624,11 +99528,11 @@ function classifyLockArtifacts(akmDir) {
|
|
|
99624
99528
|
return { removable, skipped };
|
|
99625
99529
|
}
|
|
99626
99530
|
function findWriterRelocationEntries(stashDir) {
|
|
99627
|
-
const akmDir =
|
|
99531
|
+
const akmDir = path66.join(stashDir, ".akm");
|
|
99628
99532
|
const directories = [];
|
|
99629
99533
|
for (const spec of relocationSpecs(stashDir)) {
|
|
99630
99534
|
const relativeParts = Array.isArray(spec.oldRelative) ? spec.oldRelative : [spec.oldRelative];
|
|
99631
|
-
const oldPath =
|
|
99535
|
+
const oldPath = path66.join(akmDir, ...relativeParts);
|
|
99632
99536
|
const fileCount = fileCountIfExists(oldPath);
|
|
99633
99537
|
if (fileCount === undefined || fileCount === 0)
|
|
99634
99538
|
continue;
|
|
@@ -99639,36 +99543,36 @@ function findWriterRelocationEntries(stashDir) {
|
|
|
99639
99543
|
}
|
|
99640
99544
|
function moveFile(oldFilePath, newFilePath) {
|
|
99641
99545
|
try {
|
|
99642
|
-
|
|
99546
|
+
fs55.renameSync(oldFilePath, newFilePath);
|
|
99643
99547
|
} catch (error2) {
|
|
99644
99548
|
if (error2.code !== "EXDEV")
|
|
99645
99549
|
throw error2;
|
|
99646
|
-
|
|
99647
|
-
|
|
99550
|
+
fs55.copyFileSync(oldFilePath, newFilePath);
|
|
99551
|
+
fs55.rmSync(oldFilePath, { force: true });
|
|
99648
99552
|
}
|
|
99649
99553
|
}
|
|
99650
99554
|
function moveDirectoryContents(entry) {
|
|
99651
99555
|
const errors3 = [];
|
|
99652
99556
|
let moved = 0;
|
|
99653
|
-
|
|
99557
|
+
fs55.mkdirSync(entry.newPath, { recursive: true });
|
|
99654
99558
|
let names;
|
|
99655
99559
|
try {
|
|
99656
|
-
names =
|
|
99560
|
+
names = fs55.readdirSync(entry.oldPath).sort();
|
|
99657
99561
|
} catch {
|
|
99658
99562
|
return { key: entry.key, oldPath: entry.oldPath, newPath: entry.newPath, moved: 0, errors: [] };
|
|
99659
99563
|
}
|
|
99660
99564
|
for (const name of names) {
|
|
99661
|
-
const oldFilePath =
|
|
99662
|
-
const newFilePath =
|
|
99565
|
+
const oldFilePath = path66.join(entry.oldPath, name);
|
|
99566
|
+
const newFilePath = path66.join(entry.newPath, name);
|
|
99663
99567
|
let oldStat;
|
|
99664
99568
|
try {
|
|
99665
|
-
oldStat =
|
|
99569
|
+
oldStat = fs55.lstatSync(oldFilePath);
|
|
99666
99570
|
} catch {
|
|
99667
99571
|
continue;
|
|
99668
99572
|
}
|
|
99669
99573
|
if (!oldStat.isFile())
|
|
99670
99574
|
continue;
|
|
99671
|
-
if (
|
|
99575
|
+
if (fs55.existsSync(newFilePath))
|
|
99672
99576
|
continue;
|
|
99673
99577
|
try {
|
|
99674
99578
|
moveFile(oldFilePath, newFilePath);
|
|
@@ -99681,13 +99585,13 @@ function moveDirectoryContents(entry) {
|
|
|
99681
99585
|
}
|
|
99682
99586
|
function removeIfEmptyDir(dir) {
|
|
99683
99587
|
try {
|
|
99684
|
-
if (
|
|
99685
|
-
|
|
99588
|
+
if (fs55.readdirSync(dir).length === 0)
|
|
99589
|
+
fs55.rmdirSync(dir);
|
|
99686
99590
|
} catch {}
|
|
99687
99591
|
}
|
|
99688
99592
|
function removeLockArtifact(entry) {
|
|
99689
99593
|
try {
|
|
99690
|
-
|
|
99594
|
+
fs55.rmSync(entry.path, { force: true });
|
|
99691
99595
|
return { path: entry.path, removed: true };
|
|
99692
99596
|
} catch (error2) {
|
|
99693
99597
|
return { path: entry.path, removed: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
@@ -99699,36 +99603,36 @@ function applyWriterRelocation(stashDir) {
|
|
|
99699
99603
|
const lockResults = lockArtifacts.map(removeLockArtifact);
|
|
99700
99604
|
for (const spec of relocationSpecs(stashDir)) {
|
|
99701
99605
|
const relativeParts = Array.isArray(spec.oldRelative) ? spec.oldRelative : [spec.oldRelative];
|
|
99702
|
-
removeIfEmptyDir(
|
|
99606
|
+
removeIfEmptyDir(path66.join(stashDir, ".akm", ...relativeParts));
|
|
99703
99607
|
}
|
|
99704
99608
|
return { directories: directoryResults, lockArtifacts: lockResults, skippedLocks };
|
|
99705
99609
|
}
|
|
99706
99610
|
|
|
99707
99611
|
// scripts/akm-migrate/task-migrate.ts
|
|
99708
99612
|
import { randomUUID as randomUUID7 } from "crypto";
|
|
99709
|
-
import
|
|
99613
|
+
import fs59 from "fs";
|
|
99710
99614
|
import os9 from "os";
|
|
99711
|
-
import
|
|
99615
|
+
import path69 from "path";
|
|
99712
99616
|
init_errors();
|
|
99713
99617
|
init_paths();
|
|
99714
99618
|
|
|
99715
99619
|
// scripts/akm-migrate/migrate/task-files-to-v3.ts
|
|
99716
99620
|
init_errors();
|
|
99717
99621
|
import crypto6 from "crypto";
|
|
99718
|
-
import
|
|
99719
|
-
import
|
|
99622
|
+
import fs57 from "fs";
|
|
99623
|
+
import path67 from "path";
|
|
99720
99624
|
|
|
99721
99625
|
// scripts/akm-migrate/migrate/durable-fs.ts
|
|
99722
|
-
import
|
|
99626
|
+
import fs56 from "fs";
|
|
99723
99627
|
function fsyncDirectoryPortable(directory) {
|
|
99724
99628
|
if (process.platform === "win32")
|
|
99725
99629
|
return;
|
|
99726
99630
|
try {
|
|
99727
|
-
const fd =
|
|
99631
|
+
const fd = fs56.openSync(directory, "r");
|
|
99728
99632
|
try {
|
|
99729
|
-
|
|
99633
|
+
fs56.fsyncSync(fd);
|
|
99730
99634
|
} finally {
|
|
99731
|
-
|
|
99635
|
+
fs56.closeSync(fd);
|
|
99732
99636
|
}
|
|
99733
99637
|
} catch (cause) {
|
|
99734
99638
|
const code = cause.code;
|
|
@@ -99742,25 +99646,25 @@ function migrationError(detail) {
|
|
|
99742
99646
|
return new ConfigError(`Task migration to v3 failed: ${detail}`, "INVALID_CONFIG_FILE");
|
|
99743
99647
|
}
|
|
99744
99648
|
function contained2(root2, candidate) {
|
|
99745
|
-
const relative =
|
|
99746
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
99649
|
+
const relative = path67.relative(root2, candidate);
|
|
99650
|
+
return relative === "" || !relative.startsWith("..") && !path67.isAbsolute(relative);
|
|
99747
99651
|
}
|
|
99748
99652
|
function realDirectory(filePath) {
|
|
99749
|
-
const stat =
|
|
99653
|
+
const stat = fs57.lstatSync(filePath);
|
|
99750
99654
|
if (stat.isSymbolicLink() || !stat.isDirectory())
|
|
99751
99655
|
throw migrationError(`${filePath} must be a real directory.`);
|
|
99752
|
-
return
|
|
99656
|
+
return fs57.realpathSync(filePath);
|
|
99753
99657
|
}
|
|
99754
99658
|
function snapshot(filePath) {
|
|
99755
|
-
const stat =
|
|
99659
|
+
const stat = fs57.lstatSync(filePath);
|
|
99756
99660
|
if (stat.isSymbolicLink() || !stat.isFile())
|
|
99757
99661
|
throw migrationError(`${filePath} must be a real file.`);
|
|
99758
|
-
const bytes =
|
|
99662
|
+
const bytes = fs57.readFileSync(filePath);
|
|
99759
99663
|
return Object.freeze({ bytes, mode: stat.mode & 511 });
|
|
99760
99664
|
}
|
|
99761
99665
|
function writable(filePath) {
|
|
99762
99666
|
try {
|
|
99763
|
-
|
|
99667
|
+
fs57.accessSync(filePath, fs57.constants.W_OK);
|
|
99764
99668
|
return true;
|
|
99765
99669
|
} catch {
|
|
99766
99670
|
return false;
|
|
@@ -99773,12 +99677,12 @@ function walkTasks(root2, tasksDir, out) {
|
|
|
99773
99677
|
throw migrationError(`${root2.root} resolves outside bundle ${root2.bundleId}.`);
|
|
99774
99678
|
}
|
|
99775
99679
|
const visit2 = (directory) => {
|
|
99776
|
-
const physicalDirectory =
|
|
99680
|
+
const physicalDirectory = fs57.realpathSync(directory);
|
|
99777
99681
|
if (!contained2(physicalRoot, physicalDirectory)) {
|
|
99778
99682
|
throw migrationError(`${directory} resolves outside bundle ${root2.bundleId}.`);
|
|
99779
99683
|
}
|
|
99780
|
-
for (const entry of
|
|
99781
|
-
const candidate =
|
|
99684
|
+
for (const entry of fs57.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
99685
|
+
const candidate = path67.join(directory, entry.name);
|
|
99782
99686
|
if (entry.isSymbolicLink())
|
|
99783
99687
|
throw migrationError(`task migration does not follow symbolic link ${candidate}.`);
|
|
99784
99688
|
if (entry.isDirectory()) {
|
|
@@ -99788,7 +99692,7 @@ function walkTasks(root2, tasksDir, out) {
|
|
|
99788
99692
|
if (!entry.isFile() || !entry.name.endsWith(".yml"))
|
|
99789
99693
|
continue;
|
|
99790
99694
|
const current = snapshot(candidate);
|
|
99791
|
-
const parent =
|
|
99695
|
+
const parent = path67.dirname(candidate);
|
|
99792
99696
|
out.push({
|
|
99793
99697
|
filePath: candidate,
|
|
99794
99698
|
bytes: current.bytes,
|
|
@@ -99804,9 +99708,9 @@ function walkTasks(root2, tasksDir, out) {
|
|
|
99804
99708
|
function inspectTaskToV3Files(roots) {
|
|
99805
99709
|
const files = [];
|
|
99806
99710
|
for (const root2 of [...roots].sort((a, b) => a.bundleId.localeCompare(b.bundleId))) {
|
|
99807
|
-
const tasksDir = root2.layout === "akm-task" ? root2.root :
|
|
99711
|
+
const tasksDir = root2.layout === "akm-task" ? root2.root : path67.join(root2.root, "tasks");
|
|
99808
99712
|
try {
|
|
99809
|
-
const stat =
|
|
99713
|
+
const stat = fs57.lstatSync(tasksDir);
|
|
99810
99714
|
if (stat.isSymbolicLink())
|
|
99811
99715
|
throw migrationError(`task migration does not follow symbolic link ${tasksDir}.`);
|
|
99812
99716
|
if (!stat.isDirectory())
|
|
@@ -99821,33 +99725,33 @@ function inspectTaskToV3Files(roots) {
|
|
|
99821
99725
|
return files.sort((a, b) => a.filePath.localeCompare(b.filePath));
|
|
99822
99726
|
}
|
|
99823
99727
|
function hashPath(filePath) {
|
|
99824
|
-
return crypto6.createHash("sha256").update(
|
|
99728
|
+
return crypto6.createHash("sha256").update(path67.resolve(filePath)).digest("hex").slice(0, 16);
|
|
99825
99729
|
}
|
|
99826
99730
|
function taskMigrationBackupPath(backupRoot, filePath) {
|
|
99827
|
-
return
|
|
99731
|
+
return path67.join(backupRoot, "files", `${hashPath(filePath)}-${path67.basename(filePath)}`);
|
|
99828
99732
|
}
|
|
99829
99733
|
function writeDurable(filePath, bytes, mode, exclusive = false) {
|
|
99830
|
-
|
|
99734
|
+
fs57.mkdirSync(path67.dirname(filePath), { recursive: true });
|
|
99831
99735
|
const flags = exclusive ? "wx" : "w";
|
|
99832
|
-
const fd =
|
|
99736
|
+
const fd = fs57.openSync(filePath, flags, mode);
|
|
99833
99737
|
try {
|
|
99834
|
-
|
|
99835
|
-
|
|
99738
|
+
fs57.writeFileSync(fd, bytes);
|
|
99739
|
+
fs57.fsyncSync(fd);
|
|
99836
99740
|
} finally {
|
|
99837
|
-
|
|
99741
|
+
fs57.closeSync(fd);
|
|
99838
99742
|
}
|
|
99839
|
-
|
|
99840
|
-
fsyncDirectoryPortable(
|
|
99743
|
+
fs57.chmodSync(filePath, mode);
|
|
99744
|
+
fsyncDirectoryPortable(path67.dirname(filePath));
|
|
99841
99745
|
}
|
|
99842
99746
|
function replaceAtomically(filePath, bytes, mode) {
|
|
99843
|
-
const temporary =
|
|
99747
|
+
const temporary = path67.join(path67.dirname(filePath), `.${path67.basename(filePath)}.migrate-${crypto6.randomUUID()}`);
|
|
99844
99748
|
try {
|
|
99845
99749
|
writeDurable(temporary, bytes, mode, true);
|
|
99846
|
-
|
|
99847
|
-
fsyncDirectoryPortable(
|
|
99750
|
+
fs57.renameSync(temporary, filePath);
|
|
99751
|
+
fsyncDirectoryPortable(path67.dirname(filePath));
|
|
99848
99752
|
} finally {
|
|
99849
99753
|
try {
|
|
99850
|
-
|
|
99754
|
+
fs57.unlinkSync(temporary);
|
|
99851
99755
|
} catch (cause) {
|
|
99852
99756
|
if (cause.code !== "ENOENT")
|
|
99853
99757
|
throw cause;
|
|
@@ -99886,7 +99790,7 @@ function applyTaskToV3MigrationPlan(plan, options) {
|
|
|
99886
99790
|
const current = snapshot(change.filePath);
|
|
99887
99791
|
if (!current.bytes.equals(change.after))
|
|
99888
99792
|
continue;
|
|
99889
|
-
replaceAtomically(change.filePath,
|
|
99793
|
+
replaceAtomically(change.filePath, fs57.readFileSync(taskMigrationBackupPath(options.backupRoot, change.filePath)), change.mode);
|
|
99890
99794
|
}
|
|
99891
99795
|
throw cause;
|
|
99892
99796
|
}
|
|
@@ -99896,31 +99800,31 @@ function applyTaskToV3MigrationPlan(plan, options) {
|
|
|
99896
99800
|
// scripts/akm-migrate/migrate/task-files-to-v4.ts
|
|
99897
99801
|
init_errors();
|
|
99898
99802
|
import crypto7 from "crypto";
|
|
99899
|
-
import
|
|
99900
|
-
import
|
|
99803
|
+
import fs58 from "fs";
|
|
99804
|
+
import path68 from "path";
|
|
99901
99805
|
function migrationError2(detail) {
|
|
99902
99806
|
return new ConfigError(`Task migration to v4 failed: ${detail}`, "INVALID_CONFIG_FILE");
|
|
99903
99807
|
}
|
|
99904
99808
|
function contained3(root2, candidate) {
|
|
99905
|
-
const relative =
|
|
99906
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
99809
|
+
const relative = path68.relative(root2, candidate);
|
|
99810
|
+
return relative === "" || !relative.startsWith("..") && !path68.isAbsolute(relative);
|
|
99907
99811
|
}
|
|
99908
99812
|
function realDirectory2(filePath) {
|
|
99909
|
-
const stat =
|
|
99813
|
+
const stat = fs58.lstatSync(filePath);
|
|
99910
99814
|
if (stat.isSymbolicLink() || !stat.isDirectory())
|
|
99911
99815
|
throw migrationError2(`${filePath} must be a real directory.`);
|
|
99912
|
-
return
|
|
99816
|
+
return fs58.realpathSync(filePath);
|
|
99913
99817
|
}
|
|
99914
99818
|
function snapshot2(filePath) {
|
|
99915
|
-
const stat =
|
|
99819
|
+
const stat = fs58.lstatSync(filePath);
|
|
99916
99820
|
if (stat.isSymbolicLink() || !stat.isFile())
|
|
99917
99821
|
throw migrationError2(`${filePath} must be a real file.`);
|
|
99918
|
-
const bytes =
|
|
99822
|
+
const bytes = fs58.readFileSync(filePath);
|
|
99919
99823
|
return Object.freeze({ bytes, mode: stat.mode & 511 });
|
|
99920
99824
|
}
|
|
99921
99825
|
function writable2(filePath) {
|
|
99922
99826
|
try {
|
|
99923
|
-
|
|
99827
|
+
fs58.accessSync(filePath, fs58.constants.W_OK);
|
|
99924
99828
|
return true;
|
|
99925
99829
|
} catch {
|
|
99926
99830
|
return false;
|
|
@@ -99933,12 +99837,12 @@ function walkTasks2(root2, tasksDir, out) {
|
|
|
99933
99837
|
throw migrationError2(`${root2.root} resolves outside bundle ${root2.bundleId}.`);
|
|
99934
99838
|
}
|
|
99935
99839
|
const visit2 = (directory) => {
|
|
99936
|
-
const physicalDirectory =
|
|
99840
|
+
const physicalDirectory = fs58.realpathSync(directory);
|
|
99937
99841
|
if (!contained3(physicalRoot, physicalDirectory)) {
|
|
99938
99842
|
throw migrationError2(`${directory} resolves outside bundle ${root2.bundleId}.`);
|
|
99939
99843
|
}
|
|
99940
|
-
for (const entry of
|
|
99941
|
-
const candidate =
|
|
99844
|
+
for (const entry of fs58.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
99845
|
+
const candidate = path68.join(directory, entry.name);
|
|
99942
99846
|
if (entry.isSymbolicLink())
|
|
99943
99847
|
throw migrationError2(`task migration does not follow symbolic link ${candidate}.`);
|
|
99944
99848
|
if (entry.isDirectory()) {
|
|
@@ -99948,7 +99852,7 @@ function walkTasks2(root2, tasksDir, out) {
|
|
|
99948
99852
|
if (!entry.isFile() || !entry.name.endsWith(".yml"))
|
|
99949
99853
|
continue;
|
|
99950
99854
|
const current = snapshot2(candidate);
|
|
99951
|
-
const parent =
|
|
99855
|
+
const parent = path68.dirname(candidate);
|
|
99952
99856
|
out.push({
|
|
99953
99857
|
filePath: candidate,
|
|
99954
99858
|
bytes: current.bytes,
|
|
@@ -99964,9 +99868,9 @@ function walkTasks2(root2, tasksDir, out) {
|
|
|
99964
99868
|
function inspectTaskToV4Files(roots) {
|
|
99965
99869
|
const files = [];
|
|
99966
99870
|
for (const root2 of [...roots].sort((a, b) => a.bundleId.localeCompare(b.bundleId))) {
|
|
99967
|
-
const tasksDir = root2.layout === "akm-task" ? root2.root :
|
|
99871
|
+
const tasksDir = root2.layout === "akm-task" ? root2.root : path68.join(root2.root, "tasks");
|
|
99968
99872
|
try {
|
|
99969
|
-
const stat =
|
|
99873
|
+
const stat = fs58.lstatSync(tasksDir);
|
|
99970
99874
|
if (stat.isSymbolicLink())
|
|
99971
99875
|
throw migrationError2(`task migration does not follow symbolic link ${tasksDir}.`);
|
|
99972
99876
|
if (!stat.isDirectory())
|
|
@@ -99981,33 +99885,33 @@ function inspectTaskToV4Files(roots) {
|
|
|
99981
99885
|
return files.sort((a, b) => a.filePath.localeCompare(b.filePath));
|
|
99982
99886
|
}
|
|
99983
99887
|
function hashPath2(filePath) {
|
|
99984
|
-
return crypto7.createHash("sha256").update(
|
|
99888
|
+
return crypto7.createHash("sha256").update(path68.resolve(filePath)).digest("hex").slice(0, 16);
|
|
99985
99889
|
}
|
|
99986
99890
|
function taskMigrationBackupPathV4(backupRoot, filePath) {
|
|
99987
|
-
return
|
|
99891
|
+
return path68.join(backupRoot, "files", `${hashPath2(filePath)}-${path68.basename(filePath)}`);
|
|
99988
99892
|
}
|
|
99989
99893
|
function writeDurable2(filePath, bytes, mode, exclusive = false) {
|
|
99990
|
-
|
|
99894
|
+
fs58.mkdirSync(path68.dirname(filePath), { recursive: true });
|
|
99991
99895
|
const flags = exclusive ? "wx" : "w";
|
|
99992
|
-
const fd =
|
|
99896
|
+
const fd = fs58.openSync(filePath, flags, mode);
|
|
99993
99897
|
try {
|
|
99994
|
-
|
|
99995
|
-
|
|
99898
|
+
fs58.writeFileSync(fd, bytes);
|
|
99899
|
+
fs58.fsyncSync(fd);
|
|
99996
99900
|
} finally {
|
|
99997
|
-
|
|
99901
|
+
fs58.closeSync(fd);
|
|
99998
99902
|
}
|
|
99999
|
-
|
|
100000
|
-
fsyncDirectoryPortable(
|
|
99903
|
+
fs58.chmodSync(filePath, mode);
|
|
99904
|
+
fsyncDirectoryPortable(path68.dirname(filePath));
|
|
100001
99905
|
}
|
|
100002
99906
|
function replaceAtomically2(filePath, bytes, mode) {
|
|
100003
|
-
const temporary =
|
|
99907
|
+
const temporary = path68.join(path68.dirname(filePath), `.${path68.basename(filePath)}.migrate-${crypto7.randomUUID()}`);
|
|
100004
99908
|
try {
|
|
100005
99909
|
writeDurable2(temporary, bytes, mode, true);
|
|
100006
|
-
|
|
100007
|
-
fsyncDirectoryPortable(
|
|
99910
|
+
fs58.renameSync(temporary, filePath);
|
|
99911
|
+
fsyncDirectoryPortable(path68.dirname(filePath));
|
|
100008
99912
|
} finally {
|
|
100009
99913
|
try {
|
|
100010
|
-
|
|
99914
|
+
fs58.unlinkSync(temporary);
|
|
100011
99915
|
} catch (cause) {
|
|
100012
99916
|
if (cause.code !== "ENOENT")
|
|
100013
99917
|
throw cause;
|
|
@@ -100046,7 +99950,7 @@ function applyTaskToV4MigrationPlan(plan, options) {
|
|
|
100046
99950
|
const current = snapshot2(change.filePath);
|
|
100047
99951
|
if (!current.bytes.equals(change.after))
|
|
100048
99952
|
continue;
|
|
100049
|
-
replaceAtomically2(change.filePath,
|
|
99953
|
+
replaceAtomically2(change.filePath, fs58.readFileSync(taskMigrationBackupPathV4(options.backupRoot, change.filePath)), change.mode);
|
|
100050
99954
|
}
|
|
100051
99955
|
throw cause;
|
|
100052
99956
|
}
|
|
@@ -100058,12 +99962,12 @@ function expandTilde(value) {
|
|
|
100058
99962
|
if (value === "~")
|
|
100059
99963
|
return os9.homedir();
|
|
100060
99964
|
if (value.startsWith("~/") || value.startsWith("~\\"))
|
|
100061
|
-
return
|
|
99965
|
+
return path69.join(os9.homedir(), value.slice(2));
|
|
100062
99966
|
return value;
|
|
100063
99967
|
}
|
|
100064
99968
|
function existingDirectory(target) {
|
|
100065
99969
|
try {
|
|
100066
|
-
return
|
|
99970
|
+
return fs59.statSync(target).isDirectory();
|
|
100067
99971
|
} catch (cause) {
|
|
100068
99972
|
if (cause.code === "ENOENT")
|
|
100069
99973
|
return false;
|
|
@@ -100084,21 +99988,21 @@ function taskRoots(config, resolutionBase = process.cwd()) {
|
|
|
100084
99988
|
const source = sources.get(bundleId);
|
|
100085
99989
|
if (!source)
|
|
100086
99990
|
continue;
|
|
100087
|
-
const configuredRoot = source.type === "filesystem" && source.path ?
|
|
99991
|
+
const configuredRoot = source.type === "filesystem" && source.path ? path69.resolve(resolutionBase, expandTilde(source.path)) : lockContentRootFor(bundleId, source.type);
|
|
100088
99992
|
if (!configuredRoot || !existingDirectory(configuredRoot))
|
|
100089
99993
|
continue;
|
|
100090
|
-
const bundleRoot =
|
|
99994
|
+
const bundleRoot = path69.resolve(configuredRoot);
|
|
100091
99995
|
const component = bundleComponentConfig(bundle);
|
|
100092
|
-
const componentRoot =
|
|
100093
|
-
const relative =
|
|
100094
|
-
if (relative === ".." || relative.startsWith(`..${
|
|
99996
|
+
const componentRoot = path69.resolve(bundleRoot, component?.root ?? ".");
|
|
99997
|
+
const relative = path69.relative(bundleRoot, componentRoot);
|
|
99998
|
+
if (relative === ".." || relative.startsWith(`..${path69.sep}`) || path69.isAbsolute(relative)) {
|
|
100095
99999
|
throw new ConfigError(`Task migration component root ${componentRoot} escapes bundle ${bundleId} at ${bundleRoot}.`, "INVALID_CONFIG_FILE");
|
|
100096
100000
|
}
|
|
100097
100001
|
if (!existingDirectory(componentRoot))
|
|
100098
100002
|
continue;
|
|
100099
100003
|
const adapter = component?.adapter ?? detectAdapterId(componentRoot, "");
|
|
100100
100004
|
if (!component?.adapter && adapter === "") {
|
|
100101
|
-
const flatTasks =
|
|
100005
|
+
const flatTasks = fs59.readdirSync(componentRoot, { withFileTypes: true }).filter((entry) => entry.isFile() && entry.name.endsWith(".yml")).map((entry) => entry.name).sort();
|
|
100102
100006
|
if (flatTasks.length > 0) {
|
|
100103
100007
|
throw new ConfigError(`Task migration cannot classify top-level task file(s) ${flatTasks.join(", ")} in bundle ${bundleId}; configure adapter "akm-task" or move them under tasks/.`, "INVALID_CONFIG_FILE");
|
|
100104
100008
|
}
|
|
@@ -100170,8 +100074,8 @@ function applyTaskV3Migration() {
|
|
|
100170
100074
|
const before = inspectCurrentTaskPlan();
|
|
100171
100075
|
if (before.result.taskV3Migration.changed === 0)
|
|
100172
100076
|
return before.result;
|
|
100173
|
-
const backupRoot =
|
|
100174
|
-
const backupPath =
|
|
100077
|
+
const backupRoot = path69.join(getDataDir(), "backups", "task-v3");
|
|
100078
|
+
const backupPath = path69.join(backupRoot, `${Date.now()}-${randomUUID7()}`);
|
|
100175
100079
|
const applied = applyTaskToV3MigrationPlan(before.plan, { backupRoot: backupPath });
|
|
100176
100080
|
const after = inspectCurrentTaskPlan().result;
|
|
100177
100081
|
if (after.taskV3Migration.changed > 0) {
|
|
@@ -100227,8 +100131,8 @@ function applyTaskV4Migration() {
|
|
|
100227
100131
|
const before = inspectCurrentTaskV4Plan();
|
|
100228
100132
|
if (before.result.taskV4Migration.changed === 0)
|
|
100229
100133
|
return before.result;
|
|
100230
|
-
const backupRoot =
|
|
100231
|
-
const backupPath =
|
|
100134
|
+
const backupRoot = path69.join(getDataDir(), "backups", "task-v4");
|
|
100135
|
+
const backupPath = path69.join(backupRoot, `${Date.now()}-${randomUUID7()}`);
|
|
100232
100136
|
const applied = applyTaskToV4MigrationPlan(before.plan, { backupRoot: backupPath });
|
|
100233
100137
|
const after = inspectCurrentTaskV4Plan().result;
|
|
100234
100138
|
if (after.taskV4Migration.changed > 0) {
|