akm-cli 0.9.1 → 0.9.2-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (350) hide show
  1. package/CHANGELOG.md +103 -28
  2. package/README.md +3 -1
  3. package/SECURITY.md +1 -1
  4. package/STABILITY.md +1 -1
  5. package/dist/akm +2 -2
  6. package/dist/akm-migrate +2 -2
  7. package/dist/assets/hints/cli-hints-full.md +14 -9
  8. package/dist/assets/improve-strategies/proactive-maintenance.json +1 -1
  9. package/dist/assets/improve-strategies/reflect-distill.json +1 -1
  10. package/dist/assets/models.json +35 -0
  11. package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +3 -4
  12. package/dist/assets/stash-skeleton/facts/conventions/organization.md +1 -3
  13. package/dist/assets/tasks/core/extract.yml +6 -5
  14. package/dist/assets/tasks/core/improve.yml +6 -5
  15. package/dist/assets/tasks/core/index-refresh.yml +6 -5
  16. package/dist/assets/tasks/core/sync.yml +6 -5
  17. package/dist/assets/tasks/core/version-check.yml +6 -5
  18. package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +6 -5
  19. package/dist/assets/tasks/improve/akm-improve-catchup.yml +6 -5
  20. package/dist/assets/tasks/improve/akm-improve-consolidate.yml +6 -5
  21. package/dist/assets/tasks/improve/akm-improve-frequent.yml +6 -5
  22. package/dist/assets/tasks/improve/akm-improve-nightly.yml +6 -5
  23. package/dist/cli/confirm.js +2 -2
  24. package/dist/cli/parse-args.js +3 -24
  25. package/dist/cli/retired-commands.js +1 -1
  26. package/dist/cli/shared.js +2 -2
  27. package/dist/cli.js +11 -9
  28. package/dist/commands/agent/agent-dispatch.js +55 -89
  29. package/dist/commands/agent/contribute-cli.js +12 -45
  30. package/dist/commands/command/builtin-action.js +32 -0
  31. package/dist/commands/command/command-cli.js +99 -0
  32. package/dist/commands/command/command-execution.js +308 -0
  33. package/dist/commands/command/execution-source-loader.js +176 -0
  34. package/dist/commands/command/portable-template.js +60 -0
  35. package/dist/commands/config-cli.js +10 -4
  36. package/dist/commands/env/env.js +4 -2
  37. package/dist/commands/feedback-cli.js +1 -1
  38. package/dist/commands/health/checks.js +241 -29
  39. package/dist/commands/health/html-report.js +0 -14
  40. package/dist/commands/health/report-view-model.js +0 -1
  41. package/dist/commands/health/surfaces.js +6 -7
  42. package/dist/commands/health/types.js +0 -2
  43. package/dist/commands/health.js +63 -18
  44. package/dist/commands/improve/collapse-detector.js +5 -6
  45. package/dist/commands/improve/consolidate.js +251 -214
  46. package/dist/commands/improve/distill/promote-memory.js +71 -34
  47. package/dist/commands/improve/distill/quality-gate.js +17 -5
  48. package/dist/commands/improve/distill.js +232 -155
  49. package/dist/commands/improve/eligibility.js +112 -79
  50. package/dist/commands/improve/execution.js +57 -0
  51. package/dist/commands/improve/extract-cli.js +5 -5
  52. package/dist/commands/improve/extract-prompt.js +64 -22
  53. package/dist/commands/improve/extract.js +608 -360
  54. package/dist/commands/improve/improve-strategies.js +43 -14
  55. package/dist/commands/improve/improve.js +249 -29
  56. package/dist/commands/improve/loop-stages.js +11 -17
  57. package/dist/commands/improve/memory/memory-contradiction-detect.js +90 -66
  58. package/dist/commands/improve/outcome-loop.js +22 -38
  59. package/dist/commands/improve/planner.js +134 -0
  60. package/dist/commands/improve/preparation.js +730 -409
  61. package/dist/commands/improve/reflect.js +386 -223
  62. package/dist/commands/improve/run-context.js +3 -4
  63. package/dist/commands/improve/salience.js +6 -58
  64. package/dist/commands/improve/session-asset.js +12 -12
  65. package/dist/commands/lint/index.js +101 -29
  66. package/dist/commands/migrate-cli.js +11 -69
  67. package/dist/commands/migration-tool.js +6 -9
  68. package/dist/commands/models-cli.js +27 -0
  69. package/dist/commands/proposal/drain.js +258 -186
  70. package/dist/commands/proposal/proposal-cli.js +32 -10
  71. package/dist/commands/proposal/proposal.js +2 -5
  72. package/dist/commands/proposal/propose.js +192 -172
  73. package/dist/commands/proposal/repository.js +54 -91
  74. package/dist/commands/proposal/validators/proposal-validators.js +9 -7
  75. package/dist/commands/read/curate.js +53 -22
  76. package/dist/commands/read/registry-search.js +25 -9
  77. package/dist/commands/read/remember-cli.js +14 -2
  78. package/dist/commands/read/search.js +10 -4
  79. package/dist/commands/read/show.js +139 -153
  80. package/dist/commands/registry-cli.js +16 -7
  81. package/dist/commands/remember.js +33 -18
  82. package/dist/commands/sources/add-cli.js +19 -178
  83. package/dist/commands/sources/bundle-cli.js +15 -3
  84. package/dist/commands/sources/dangerous-env-audit.js +135 -0
  85. package/dist/commands/sources/info.js +2 -1
  86. package/dist/commands/sources/installed-stashes.js +901 -177
  87. package/dist/commands/sources/schema-repair.js +174 -95
  88. package/dist/commands/sources/self-update.js +30 -74
  89. package/dist/commands/sources/source-add.js +3 -5
  90. package/dist/commands/sources/sources-cli.js +2 -15
  91. package/dist/commands/sources/update-transaction.js +220 -0
  92. package/dist/commands/tasks/tasks-cli.js +3 -3
  93. package/dist/commands/tasks/tasks.js +736 -317
  94. package/dist/commands/workflow-cli.js +2 -2
  95. package/dist/core/adapter/adapters/agent-skills-adapter.js +3 -0
  96. package/dist/core/adapter/adapters/akm-adapter.js +85 -35
  97. package/dist/core/adapter/adapters/akm-lint.js +54 -39
  98. package/dist/core/adapter/adapters/akm-metadata.js +45 -45
  99. package/dist/core/adapter/adapters/akm-task-adapter.js +32 -49
  100. package/dist/core/adapter/adapters/akm-workflow-adapter.js +38 -23
  101. package/dist/core/adapter/adapters/dotenv-adapter.js +30 -1
  102. package/dist/core/adapter/adapters/generic-files-adapter.js +11 -0
  103. package/dist/core/adapter/adapters/index.js +0 -9
  104. package/dist/core/adapter/adapters/llm-wiki-adapter.js +4 -0
  105. package/dist/core/adapter/adapters/okf-adapter.js +4 -0
  106. package/dist/core/adapter/adapters/opencode-adapter.js +5 -8
  107. package/dist/core/adapter/adapters/tool-dir-shared.js +63 -6
  108. package/dist/core/adapter/adapters/website-snapshot-adapter.js +4 -0
  109. package/dist/core/adapter/execution-source.js +308 -0
  110. package/dist/core/adapter/recognize-match.js +36 -13
  111. package/dist/core/adapter/registry.js +0 -9
  112. package/dist/core/asset/stash-meta.js +94 -4
  113. package/dist/core/common.js +6 -11
  114. package/dist/core/config/config-io.js +3 -3
  115. package/dist/core/config/config-schema.js +18 -40
  116. package/dist/core/config/config-sources.js +11 -21
  117. package/dist/core/config/config-walker.js +31 -13
  118. package/dist/core/config/config.js +23 -26
  119. package/dist/core/config/schema/engines.js +8 -7
  120. package/dist/core/config/schema/improve-processes.js +29 -5
  121. package/dist/core/config/schema/index-config.js +0 -27
  122. package/dist/core/config/schema/primitives.js +1 -23
  123. package/dist/core/config/schema/sources-bundles.js +13 -16
  124. package/dist/core/errors.js +2 -0
  125. package/dist/core/events.js +68 -32
  126. package/dist/core/extra-params.js +1 -0
  127. package/dist/core/improve-result.js +315 -0
  128. package/dist/core/lesson-lint.js +0 -6
  129. package/dist/core/maintenance-barrier.js +4 -4
  130. package/dist/core/network-policy.js +152 -0
  131. package/dist/core/paths.js +1 -1
  132. package/dist/core/recognition-util.js +4 -4
  133. package/dist/core/registry-url.js +456 -0
  134. package/dist/core/state/migrations.js +161 -47
  135. package/dist/core/state-db.js +453 -80
  136. package/dist/core/system-error.js +32 -0
  137. package/dist/core/time.js +2 -12
  138. package/dist/core/write-source.js +0 -18
  139. package/dist/execution/directory-identity.js +52 -0
  140. package/dist/execution/executable-identity.js +107 -0
  141. package/dist/execution/guarded-source.js +398 -0
  142. package/dist/execution/json.js +95 -0
  143. package/dist/{commands/health/types-session-log.js → execution/limits.js} +2 -1
  144. package/dist/execution/record.js +55 -0
  145. package/dist/execution/resolved-request.js +730 -0
  146. package/dist/execution/source.js +320 -0
  147. package/dist/indexer/bundle-identity-guard.js +5 -4
  148. package/dist/indexer/db/graph-db.js +33 -0
  149. package/dist/indexer/graph/graph-boost.js +3 -4
  150. package/dist/indexer/graph/graph-extraction.js +562 -373
  151. package/dist/indexer/index-written-assets.js +78 -39
  152. package/dist/indexer/indexer.js +471 -432
  153. package/dist/indexer/installations.js +6 -0
  154. package/dist/indexer/lookup/adapter-concept-owner.js +283 -0
  155. package/dist/indexer/materialize-embeddings.js +155 -0
  156. package/dist/indexer/passes/memory-inference.js +227 -174
  157. package/dist/indexer/passes/metadata.js +263 -118
  158. package/dist/indexer/scan/doc-to-entry.js +7 -10
  159. package/dist/indexer/scan/drain-dir.js +51 -23
  160. package/dist/indexer/search/db-search.js +156 -50
  161. package/dist/indexer/search/fts-query.js +40 -40
  162. package/dist/indexer/search/ranking.js +36 -1
  163. package/dist/indexer/search/search-attribution.js +3 -1
  164. package/dist/indexer/search/search-fields.js +23 -14
  165. package/dist/indexer/search/search-hit-enrichers.js +1 -1
  166. package/dist/indexer/search/search-source.js +7 -16
  167. package/dist/indexer/search/semantic-status.js +10 -1
  168. package/dist/indexer/usage/show-usage.js +105 -0
  169. package/dist/indexer/usage/usage-events.js +7 -2
  170. package/dist/indexer/walk/matchers.js +40 -10
  171. package/dist/indexer/walk/path-resolver.js +5 -2
  172. package/dist/indexer/walk/walker.js +20 -2
  173. package/dist/integrations/agent/builder-shared.js +3 -6
  174. package/dist/integrations/agent/conversation-fallback.js +16 -0
  175. package/dist/integrations/agent/engine-resolution.js +87 -87
  176. package/dist/integrations/agent/execution-cascade.js +566 -0
  177. package/dist/integrations/agent/execution-definitions.js +211 -0
  178. package/dist/integrations/agent/execution-lowering.js +811 -0
  179. package/dist/integrations/agent/execution-preparation.js +67 -0
  180. package/dist/integrations/agent/index.js +0 -2
  181. package/dist/integrations/agent/inline-execution.js +74 -0
  182. package/dist/integrations/agent/model-map.js +515 -0
  183. package/dist/integrations/agent/persona-fallback.js +30 -0
  184. package/dist/integrations/agent/request-lowering.js +186 -0
  185. package/dist/integrations/agent/runner-dispatch.js +230 -37
  186. package/dist/integrations/agent/runner.js +12 -83
  187. package/dist/integrations/harnesses/aider/agent-builder.js +8 -0
  188. package/dist/integrations/harnesses/aider/index.js +0 -1
  189. package/dist/integrations/harnesses/amazonq/agent-builder.js +8 -0
  190. package/dist/integrations/harnesses/amazonq/index.js +0 -1
  191. package/dist/integrations/harnesses/claude/agent-builder.js +14 -1
  192. package/dist/integrations/harnesses/claude/index.js +1 -5
  193. package/dist/integrations/harnesses/claude/session-log.js +3 -33
  194. package/dist/integrations/harnesses/codex/agent-builder.js +8 -0
  195. package/dist/integrations/harnesses/codex/index.js +0 -1
  196. package/dist/integrations/harnesses/copilot/agent-builder.js +8 -0
  197. package/dist/integrations/harnesses/copilot/index.js +0 -1
  198. package/dist/integrations/harnesses/gemini/agent-builder.js +8 -0
  199. package/dist/integrations/harnesses/gemini/index.js +0 -1
  200. package/dist/integrations/harnesses/index.js +4 -44
  201. package/dist/integrations/harnesses/opencode/agent-builder.js +16 -9
  202. package/dist/integrations/harnesses/opencode/index.js +0 -2
  203. package/dist/integrations/harnesses/opencode/session-log.js +14 -204
  204. package/dist/integrations/harnesses/opencode-sdk/harness.js +12 -1
  205. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +40 -42
  206. package/dist/integrations/harnesses/openhands/agent-builder.js +8 -0
  207. package/dist/integrations/harnesses/openhands/index.js +0 -1
  208. package/dist/integrations/harnesses/pi/agent-builder.js +8 -0
  209. package/dist/integrations/harnesses/pi/index.js +0 -1
  210. package/dist/integrations/harnesses/shared.js +0 -1
  211. package/dist/integrations/harnesses/types.js +1 -3
  212. package/dist/integrations/lockfile.js +82 -79
  213. package/dist/integrations/session-logs/index.js +6 -17
  214. package/dist/integrations/session-logs/provider-base.js +1 -29
  215. package/dist/llm/client.js +10 -5
  216. package/dist/llm/embedder.js +6 -7
  217. package/dist/llm/embedders/local.js +37 -88
  218. package/dist/llm/embedders/types.js +1 -1
  219. package/dist/llm/graph-extract.js +75 -50
  220. package/dist/llm/index-passes.js +43 -5
  221. package/dist/llm/memory-infer.js +8 -6
  222. package/dist/llm/metadata-enhance.js +5 -3
  223. package/dist/llm/structured-call.js +122 -25
  224. package/dist/output/format-exempt.js +1 -1
  225. package/dist/output/render-registry.js +0 -16
  226. package/dist/output/renderers.js +12 -7
  227. package/dist/output/shapes/curate.js +1 -0
  228. package/dist/output/shapes/helpers.js +10 -2
  229. package/dist/output/shapes/passthrough.js +2 -0
  230. package/dist/output/text/command-format.js +31 -33
  231. package/dist/output/text/health-format.js +1 -29
  232. package/dist/output/text/migrate.js +6 -56
  233. package/dist/output/text/proposal-format.js +16 -1
  234. package/dist/output/text/workflow-format.js +16 -0
  235. package/dist/registry/network.js +279 -0
  236. package/dist/registry/pinned-request-helper.js +247 -0
  237. package/dist/registry/pinned-transport.js +717 -0
  238. package/dist/registry/providers/skills-sh.js +18 -6
  239. package/dist/registry/providers/static-index.js +20 -7
  240. package/dist/registry/resolve.js +53 -28
  241. package/dist/scripts/akm-migrate-node.js +19334 -52269
  242. package/dist/scripts/akm-migrate.js +19270 -51612
  243. package/dist/setup/registry-stash-loader.js +64 -20
  244. package/dist/setup/semantic-assets.js +9 -34
  245. package/dist/setup/setup.js +12 -30
  246. package/dist/setup/source-identity.js +17 -0
  247. package/dist/setup/steps/sources.js +36 -15
  248. package/dist/setup/steps/tasks.js +39 -11
  249. package/dist/sources/providers/git-provider.js +3 -3
  250. package/dist/sources/providers/npm.js +2 -2
  251. package/dist/sources/providers/provider-utils.js +4 -3
  252. package/dist/sources/providers/website.js +11 -7
  253. package/dist/sources/snapshot-fetchers/host-guard.js +9 -136
  254. package/dist/sources/snapshot-fetchers/website-ingest.js +25 -109
  255. package/dist/sources/website-url.js +73 -0
  256. package/dist/storage/engines/sqlite-migrations.js +81 -26
  257. package/dist/storage/managed-db.js +27 -24
  258. package/dist/storage/repositories/events-repository.js +3 -0
  259. package/dist/storage/repositories/index-connection.js +42 -10
  260. package/dist/storage/repositories/index-entries-repository.js +203 -229
  261. package/dist/storage/repositories/index-entry-mapper.js +8 -12
  262. package/dist/storage/repositories/index-entry-schema.js +255 -0
  263. package/dist/storage/repositories/index-fts-repository.js +64 -71
  264. package/dist/storage/repositories/index-llm-cache-repository.js +8 -13
  265. package/dist/storage/repositories/index-meta-repository.js +0 -11
  266. package/dist/storage/repositories/index-schema.js +74 -350
  267. package/dist/storage/repositories/index-utility-repository.js +12 -17
  268. package/dist/storage/repositories/index-vec-repository.js +56 -7
  269. package/dist/storage/repositories/proposals-repository.js +4 -127
  270. package/dist/storage/repositories/registry-cache.js +2 -1
  271. package/dist/storage/repositories/task-history-repository.js +20 -40
  272. package/dist/storage/repositories/workflow-runs-repository.js +228 -129
  273. package/dist/storage/sqlite-read-snapshot.js +148 -0
  274. package/dist/tasks/backends/cron.js +170 -42
  275. package/dist/tasks/backends/index.js +1 -1
  276. package/dist/tasks/backends/launchd.js +787 -202
  277. package/dist/tasks/backends/schtasks.js +282 -83
  278. package/dist/tasks/embedded.js +7 -7
  279. package/dist/tasks/frozen-script.js +50 -0
  280. package/dist/tasks/resolve-akm-bin.js +5 -1
  281. package/dist/tasks/runner.js +239 -251
  282. package/dist/tasks/runtime-v3.js +281 -0
  283. package/dist/tasks/scheduler-binding.js +272 -0
  284. package/dist/tasks/scheduler-invocation.js +57 -43
  285. package/dist/tasks/scheduler-sync.js +654 -0
  286. package/dist/tasks/source-v3.js +752 -0
  287. package/dist/tasks/standalone-script-entry.js +5 -0
  288. package/dist/tasks/task-id.js +29 -0
  289. package/dist/workflows/authoring/authoring.js +15 -32
  290. package/dist/workflows/exec/dispatch-redaction.js +14 -8
  291. package/dist/workflows/exec/exec-unit.js +7 -28
  292. package/dist/workflows/exec/frozen-judge.js +57 -89
  293. package/dist/workflows/exec/lowering-notices.js +23 -0
  294. package/dist/workflows/exec/native-executor.js +301 -458
  295. package/dist/workflows/exec/param-secrets.js +4 -3
  296. package/dist/workflows/exec/run-workflow.js +26 -32
  297. package/dist/workflows/exec/step-work.js +105 -109
  298. package/dist/workflows/exec/unit-dispatch.js +103 -27
  299. package/dist/workflows/exec/unit-writer.js +3 -3
  300. package/dist/workflows/exec/worktree.js +2 -2
  301. package/dist/workflows/ir/compile.js +86 -72
  302. package/dist/workflows/ir/environment-v4.js +328 -0
  303. package/dist/workflows/ir/freeze-v4.js +122 -0
  304. package/dist/workflows/ir/plan-hash.js +13 -7
  305. package/dist/workflows/ir/schema-v4.js +525 -0
  306. package/dist/workflows/ir/schema.js +25 -284
  307. package/dist/workflows/ir/source-freeze-v4.js +506 -0
  308. package/dist/workflows/parser.js +27 -24
  309. package/dist/workflows/program/schema.js +1 -2
  310. package/dist/workflows/renderer.js +42 -29
  311. package/dist/workflows/resource-limits.js +4 -5
  312. package/dist/workflows/runtime/agent-identity.js +11 -13
  313. package/dist/workflows/runtime/plan-classifier.js +8 -8
  314. package/dist/workflows/runtime/runs.js +27 -43
  315. package/dist/workflows/runtime/workflow-asset-loader.js +45 -205
  316. package/dist/workflows/source-files.js +373 -0
  317. package/dist/workflows/source-ir/compile.js +196 -0
  318. package/dist/workflows/source-ir/github-yaml.js +577 -0
  319. package/dist/workflows/source-ir/ordering.js +38 -0
  320. package/dist/workflows/source-ir/program.js +50 -0
  321. package/dist/workflows/source-ir/result.js +26 -0
  322. package/dist/workflows/source-ir/schema.js +772 -0
  323. package/dist/workflows/source-ir/semantics.js +242 -0
  324. package/dist/workflows/source-ir/uses.js +14 -0
  325. package/docs/README.md +2 -0
  326. package/docs/migration/README.md +3 -1
  327. package/docs/migration/release-notes/0.9.2.md +55 -0
  328. package/docs/migration/release-notes/README.md +5 -0
  329. package/docs/migration/v0.8-to-v0.9.md +76 -1077
  330. package/docs/migration/v0.9.0-troubleshooting.md +104 -516
  331. package/docs/migration/v0.9.1-to-v0.9.2.md +150 -0
  332. package/docs/reference/README.md +1 -0
  333. package/docs/reference/cli.md +230 -98
  334. package/docs/reference/configuration.md +159 -36
  335. package/docs/reference/data-and-telemetry.md +19 -1
  336. package/docs/reference/supported-formats.md +23 -3
  337. package/docs/reference/tasks.md +182 -0
  338. package/docs/reference/workflow-schema.md +91 -40
  339. package/docs/reference/workflows.md +33 -6
  340. package/package.json +10 -6
  341. package/schemas/akm-config.json +372 -224
  342. package/schemas/akm-task.json +324 -80
  343. package/schemas/akm-workflow.json +6 -9
  344. package/dist/core/migration-operation.js +0 -75
  345. package/dist/integrations/agent/model-aliases.js +0 -74
  346. package/dist/tasks/parser.js +0 -380
  347. package/dist/tasks/schema.js +0 -123
  348. package/dist/tasks/validator.js +0 -80
  349. package/dist/workflows/ir/freeze.js +0 -320
  350. package/dist/workflows/runtime/document-cache.js +0 -13
@@ -1,9 +1,8 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
- import { lookup as dnsLookup } from "node:dns/promises";
5
- import { isIP } from "node:net";
6
4
  import { fetchWithRetry, readBodyWithByteCap } from "../../core/common.js";
5
+ import { bareHostname, classifyNetworkAddress, classifyNetworkHostname, resolveHostnameAddresses, } from "../../core/network-policy.js";
7
6
  /** Redirect budget for a single guarded request chain. */
8
7
  const MAX_GUARDED_REDIRECTS = 8;
9
8
  export function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
@@ -22,10 +21,6 @@ export function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
22
21
  throw new ErrorType(`Refusing to fetch non-public website host: ${parsedUrl.hostname}`);
23
22
  }
24
23
  }
25
- async function defaultResolveHostname(hostname) {
26
- const records = await dnsLookup(hostname, { all: true });
27
- return records.map((record) => record.address);
28
- }
29
24
  /**
30
25
  * Resolve-then-validate SSRF guard against DNS rebinding / private-range
31
26
  * bypasses. {@link assertWebsiteRequestUrl} only rejects IP-literal and
@@ -46,18 +41,16 @@ async function defaultResolveHostname(hostname) {
46
41
  export async function assertResolvedHostAllowed(hostname, options) {
47
42
  if (options?.allowPrivateHosts === true)
48
43
  return;
49
- const bare = stripIpv6Brackets(hostname.toLowerCase());
50
- const literalVersion = isIP(bare);
51
- if (literalVersion !== 0) {
52
- const forbidden = literalVersion === 4 ? isForbiddenIpv4(bare) : isForbiddenIpv6(bare);
53
- if (forbidden)
44
+ const bare = bareHostname(hostname.toLowerCase());
45
+ const literalClass = classifyNetworkAddress(bare);
46
+ if (literalClass !== "invalid") {
47
+ if (literalClass !== "public")
54
48
  throw new Error(`Refusing to fetch non-public website host: ${hostname}`);
55
49
  return;
56
50
  }
57
- const resolve = options?.resolveHostname ?? defaultResolveHostname;
58
51
  let addresses;
59
52
  try {
60
- addresses = await resolve(bare);
53
+ addresses = await resolveHostnameAddresses(bare, options?.resolveHostname);
61
54
  }
62
55
  catch {
63
56
  throw new Error(`Refusing to fetch ${hostname}: DNS resolution failed`);
@@ -66,138 +59,18 @@ export async function assertResolvedHostAllowed(hostname, options) {
66
59
  throw new Error(`Refusing to fetch ${hostname}: hostname resolved to no addresses`);
67
60
  }
68
61
  for (const address of addresses) {
69
- const version = isIP(address);
70
- const forbidden = version === 4 ? isForbiddenIpv4(address) : version === 6 ? isForbiddenIpv6(stripIpv6Brackets(address)) : true;
71
- if (forbidden) {
62
+ if (classifyNetworkAddress(address) !== "public") {
72
63
  throw new Error(`Refusing to fetch ${hostname}: resolves to non-public or unparseable address ${address}`);
73
64
  }
74
65
  }
75
66
  }
76
- // WHATWG URL.hostname wraps IPv6 literals in brackets (e.g. "[::1]"), but
77
- // node:net's isIP() only recognizes the bare address form and returns 0 for
78
- // anything bracketed — silently skipping all IPv6 forbidden-host checks
79
- // below for every hostname parsed off a URL. Strip the brackets before any
80
- // isIP()/isForbiddenIpv6() call so those checks actually run.
81
- function stripIpv6Brackets(hostname) {
82
- return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
83
- }
84
67
  function isForbiddenWebsiteHostname(hostname, options) {
85
68
  if (options?.allowPrivateHosts === true)
86
69
  return false;
87
- if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal") {
88
- return true;
89
- }
90
- const bareHostname = stripIpv6Brackets(hostname);
91
- const ipVersion = isIP(bareHostname);
92
- if (ipVersion === 4)
93
- return isForbiddenIpv4(bareHostname);
94
- if (ipVersion === 6)
95
- return isForbiddenIpv6(bareHostname);
96
- return false;
70
+ return classifyNetworkHostname(hostname) !== "public";
97
71
  }
98
72
  export function isLoopbackWebsiteHostname(hostname) {
99
- if (hostname === "localhost" || hostname.endsWith(".localhost"))
100
- return true;
101
- const bareHostname = stripIpv6Brackets(hostname);
102
- const ipVersion = isIP(bareHostname);
103
- if (ipVersion === 4)
104
- return bareHostname.startsWith("127.");
105
- if (ipVersion === 6)
106
- return bareHostname === "::1";
107
- return false;
108
- }
109
- function isForbiddenIpv4(hostname) {
110
- const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
111
- const [a = -1, b = -1, c = -1, d = -1] = parts;
112
- if (parts.length !== 4 || [a, b, c, d].some((part) => !Number.isInteger(part) || part < 0 || part > 255)) {
113
- return true;
114
- }
115
- return (a === 0 ||
116
- a === 10 ||
117
- (a === 100 && b >= 64 && b <= 127) ||
118
- a === 127 ||
119
- (a === 169 && b === 254) ||
120
- (a === 172 && b >= 16 && b <= 31) ||
121
- (a === 192 && b === 0 && c === 0 && d !== 9 && d !== 10) ||
122
- (a === 192 && b === 0 && c === 2) ||
123
- (a === 192 && b === 168) ||
124
- (a === 192 && b === 88 && c === 99) ||
125
- (a === 198 && (b === 18 || b === 19)) ||
126
- (a === 198 && b === 51 && c === 100) ||
127
- (a === 203 && b === 0 && c === 113) ||
128
- a >= 224);
129
- }
130
- function parseIpv6Words(hostname) {
131
- let normalized;
132
- try {
133
- normalized = stripIpv6Brackets(new URL(`http://[${stripIpv6Brackets(hostname)}]/`).hostname).toLowerCase();
134
- }
135
- catch {
136
- return null;
137
- }
138
- const halves = normalized.split("::");
139
- if (halves.length > 2)
140
- return null;
141
- const left = halves[0] ? halves[0].split(":") : [];
142
- const right = halves[1] ? halves[1].split(":") : [];
143
- const missing = 8 - left.length - right.length;
144
- if ((halves.length === 1 && missing !== 0) || (halves.length === 2 && missing < 1))
145
- return null;
146
- const parts = halves.length === 2 ? [...left, ...Array(missing).fill("0"), ...right] : left;
147
- if (parts.length !== 8 || parts.some((part) => !/^[0-9a-f]{1,4}$/.test(part)))
148
- return null;
149
- return parts.map((part) => Number.parseInt(part, 16));
150
- }
151
- function ipv6Value(hostname) {
152
- const words = parseIpv6Words(hostname);
153
- if (!words)
154
- return null;
155
- return words.reduce((value, word) => (value << 16n) | BigInt(word), 0n);
156
- }
157
- const FORBIDDEN_IPV6_RANGES = [
158
- ["64:ff9b:1::", 48],
159
- ["100::", 64],
160
- ["2001::", 32],
161
- ["2001:2::", 48],
162
- ["2001:10::", 28],
163
- ["2001:20::", 28],
164
- ["2001:db8::", 32],
165
- ["3fff::", 20],
166
- ["5f00::", 16],
167
- ["fc00::", 7],
168
- ["fe80::", 10],
169
- ["fec0::", 10],
170
- ["ff00::", 8],
171
- ];
172
- function embeddedIpv4Address(words) {
173
- const firstSixZero = words.slice(0, 6).every((word) => word === 0);
174
- const mapped = words.slice(0, 5).every((word) => word === 0) && words[5] === 0xffff;
175
- const translated = words.slice(0, 4).every((word) => word === 0) && words[4] === 0xffff && words[5] === 0;
176
- const wellKnownNat64 = words[0] === 0x64 && words[1] === 0xff9b && words.slice(2, 6).every((word) => word === 0);
177
- if (words[0] === 0x2002)
178
- return ipv4FromHextets(words[1] ?? 0, words[2] ?? 0);
179
- if (!firstSixZero && !mapped && !translated && !wellKnownNat64)
180
- return null;
181
- return ipv4FromHextets(words[6] ?? 0, words[7] ?? 0);
182
- }
183
- function ipv4FromHextets(high, low) {
184
- return `${(high >> 8) & 0xff}.${high & 0xff}.${(low >> 8) & 0xff}.${low & 0xff}`;
185
- }
186
- function isForbiddenIpv6(hostname) {
187
- const words = parseIpv6Words(hostname);
188
- const value = ipv6Value(hostname);
189
- if (!words || value === null)
190
- return true;
191
- const embeddedIpv4 = embeddedIpv4Address(words);
192
- if (embeddedIpv4)
193
- return isForbiddenIpv4(embeddedIpv4);
194
- return FORBIDDEN_IPV6_RANGES.some(([prefix, bits]) => {
195
- const prefixValue = ipv6Value(prefix);
196
- if (prefixValue === null)
197
- return true;
198
- const shift = BigInt(128 - bits);
199
- return value >> shift === prefixValue >> shift;
200
- });
73
+ return classifyNetworkHostname(hostname) === "loopback";
201
74
  }
202
75
  function redirectInit(init, currentUrl, nextUrl) {
203
76
  if (new URL(currentUrl).origin === new URL(nextUrl).origin)
@@ -1,18 +1,18 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
- import { createHash, randomBytes } from "node:crypto";
4
+ import { randomBytes } from "node:crypto";
5
5
  import fs from "node:fs";
6
6
  import path from "node:path";
7
7
  import { fetchWithRetry, isWithin, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir, todayIso, } from "../../core/common.js";
8
8
  import { ConfigError, UsageError } from "../../core/errors.js";
9
- import { getRegistryIndexCacheDir } from "../../core/paths.js";
10
9
  import { warn, warnVerbose } from "../../core/warn.js";
11
10
  import { withFreshnessCache } from "../freshness.js";
12
11
  import { sanitizeString } from "../providers/provider-utils.js";
12
+ import { getWebsiteCachePaths, normalizeSiteUrl, validateWebsiteInputUrl, validateWebsiteUrl, } from "../website-url.js";
13
13
  import { htmlToMarkdownAndLinks } from "./content-extract.js";
14
14
  import { escapeMarkdownStructure } from "./fetcher-util.js";
15
- import { assertResolvedHostAllowed, assertWebsiteRequestUrl, isLoopbackWebsiteHostname, } from "./host-guard.js";
15
+ import { assertResolvedHostAllowed, assertWebsiteRequestUrl } from "./host-guard.js";
16
16
  import { loadWikiSnapshotFetchers } from "./registry.js";
17
17
  import { createAllowAllRobotsPolicy, createRobotsPolicy, isPathAllowedByRobots, ROBOTS_BODY_TIMEOUT_MS, ROBOTS_BYTE_CAP, } from "./robots.js";
18
18
  /** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
@@ -37,25 +37,15 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
37
37
  */
38
38
  const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
39
39
  const WEBSITE_MAX_REDIRECTS = 8;
40
- /**
41
- * Coerces the user-facing `crawlTimeoutMs` option.
42
- *
43
- * Returns `null` for an explicit opt-out (`false`, or `0`), the configured
44
- * number of milliseconds when positive, and `undefined` to mean "unset, use
45
- * the default". Anything else is ignored rather than failing a crawl over a
46
- * malformed knob.
47
- */
48
- function coerceCrawlTimeoutMs(value) {
49
- if (value === false || value === 0)
50
- return null;
51
- if (value === true || value === undefined || value === null)
52
- return undefined;
53
- const parsed = typeof value === "number" ? value : typeof value === "string" ? Number.parseInt(value, 10) : Number.NaN;
54
- if (!Number.isFinite(parsed))
40
+ /** Resolve the exact persisted crawl timeout; zero explicitly disables it. */
41
+ export function resolveCrawlTimeoutMs(value) {
42
+ if (value === undefined)
55
43
  return undefined;
56
- if (parsed <= 0)
44
+ if (value === 0)
57
45
  return null;
58
- return parsed;
46
+ if (typeof value === "number" && Number.isInteger(value) && value > 0)
47
+ return value;
48
+ throw new ConfigError(`Invalid value for crawlTimeoutMs: expected a non-negative integer, got ${JSON.stringify(value)}.`);
59
49
  }
60
50
  /**
61
51
  * How many times a URL may be pushed back for not fitting its origin's
@@ -70,19 +60,6 @@ const MAX_CRAWL_DEFERRALS = 3;
70
60
  * wall-clock cap elapses.
71
61
  */
72
62
  const WEBSITE_PAGE_BODY_TIMEOUT_MS = 30_000;
73
- export function shouldAllowPrivateWebsiteHostsForTests() {
74
- return process.env.BUN_TEST === "1" || process.env.NODE_ENV === "test";
75
- }
76
- export function shouldAllowPrivateWebsiteUrlForTests(rawUrl) {
77
- if (!shouldAllowPrivateWebsiteHostsForTests())
78
- return false;
79
- try {
80
- return isLoopbackWebsiteHostname(new URL(rawUrl).hostname.toLowerCase());
81
- }
82
- catch {
83
- return false;
84
- }
85
- }
86
63
  function resolveFetcherStashDir(explicitStashDir) {
87
64
  if (explicitStashDir)
88
65
  return explicitStashDir;
@@ -93,19 +70,10 @@ function resolveFetcherStashDir(explicitStashDir) {
93
70
  return null;
94
71
  }
95
72
  }
96
- export function getWebsiteCachePaths(siteUrl) {
97
- const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
98
- const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
99
- return {
100
- rootDir,
101
- stashDir: path.join(rootDir, "stash"),
102
- manifestPath: path.join(rootDir, "manifest.json"),
103
- };
104
- }
105
73
  export async function ensureWebsiteMirror(config, options) {
106
74
  const rawUrl = config.url ?? "";
107
75
  const normalizedUrl = validateWebsiteUrl(rawUrl, { allowPrivateHosts: options?.allowPrivateHosts });
108
- const cachePaths = getWebsiteCachePaths(normalizedUrl);
76
+ const cachePaths = getWebsiteCachePaths(normalizedUrl, options?.cacheRootDir);
109
77
  const requireStashDir = options?.requireStashDir === true;
110
78
  await withFreshnessCache({
111
79
  markerPath: cachePaths.manifestPath,
@@ -117,12 +85,12 @@ export async function ensureWebsiteMirror(config, options) {
117
85
  fs.mkdirSync(cachePaths.rootDir, { recursive: true });
118
86
  await scrapeWebsiteToStash(normalizedUrl, cachePaths.stashDir, {
119
87
  fetcherStashDir: resolveFetcherStashDir(),
120
- maxPages: coercePositiveInt(config.options?.maxPages, MAX_PAGES_DEFAULT),
121
- maxDepth: coercePositiveInt(config.options?.maxDepth, MAX_DEPTH_DEFAULT),
122
- respectRobots: coerceRespectRobots(config.options?.respectRobots),
88
+ maxPages: resolvePositiveInt(config.options?.maxPages, MAX_PAGES_DEFAULT, "maxPages"),
89
+ maxDepth: resolvePositiveInt(config.options?.maxDepth, MAX_DEPTH_DEFAULT, "maxDepth"),
90
+ respectRobots: resolveRespectRobots(config.options?.respectRobots),
123
91
  allowPrivateHosts: options?.allowPrivateHosts,
124
92
  wallClockCapMs: options?.wallClockCapMs,
125
- crawlTimeoutMs: coerceCrawlTimeoutMs(config.options?.crawlTimeoutMs),
93
+ crawlTimeoutMs: resolveCrawlTimeoutMs(config.options?.crawlTimeoutMs),
126
94
  resolveSecret: options?.resolveSecret,
127
95
  // As-supplied, pre-normalization start URL (see crawlWebsite's
128
96
  // `rawStartUrl` doc comment): threaded through purely for the C-02
@@ -544,7 +512,7 @@ async function crawlWebsite(startUrl, options) {
544
512
  const visited = new Set();
545
513
  const pages = [];
546
514
  // Precedence: the test-only seam, then the user's `crawlTimeoutMs`, then the
547
- // default. `crawlTimeoutMs: 0` / `false` disables the cap outright, for a
515
+ // default. `crawlTimeoutMs: 0` disables the cap outright, for a
548
516
  // deliberately long-running crawl the user is willing to babysit.
549
517
  const configuredCapMs = options.crawlTimeoutMs === null ? null : (options.crawlTimeoutMs ?? WEBSITE_CRAWL_WALL_CLOCK_MS);
550
518
  const wallClockCapMs = options.wallClockCapMs ?? configuredCapMs;
@@ -866,28 +834,13 @@ export async function loadRobotsTxt(robotsUrl, options) {
866
834
  return { kind: "unavailable" };
867
835
  }
868
836
  }
869
- /**
870
- * Coerces `SourceConfigEntry.options.respectRobots` to a boolean. The bundle
871
- * descriptor is boolean-validated at config load (schema), but the legacy
872
- * `sources[].options` bag is `z.record(z.unknown())` and accepts anything, so
873
- * the runtime read still validates. A misspelled non-boolean opt-out fails
874
- * loudly (`ConfigError`) rather than silently defaulting either way — the
875
- * user would otherwise think robots.txt handling is something other than
876
- * what akm is actually doing (spec §4.7).
877
- */
878
- export function coerceRespectRobots(value) {
879
- if (value === undefined || value === null)
837
+ /** Resolve the exact persisted robots policy. */
838
+ export function resolveRespectRobots(value) {
839
+ if (value === undefined)
880
840
  return true;
881
841
  if (typeof value === "boolean")
882
842
  return value;
883
- if (typeof value === "string") {
884
- const normalized = value.trim().toLowerCase();
885
- if (normalized === "true")
886
- return true;
887
- if (normalized === "false")
888
- return false;
889
- }
890
- throw new ConfigError(`Invalid value for respectRobots: expected a boolean (or "true"/"false"), got ${JSON.stringify(value)}.`);
843
+ throw new ConfigError(`Invalid value for respectRobots: expected a boolean, got ${JSON.stringify(value)}.`);
891
844
  }
892
845
  function buildMarkdownSnapshot(page, slug, tags) {
893
846
  const title = sanitizeString(page.title, 200) || slug;
@@ -917,41 +870,6 @@ function buildMarkdownSnapshot(page, slug, tags) {
917
870
  "",
918
871
  ].join("\n");
919
872
  }
920
- export function validateWebsiteUrl(rawUrl, options) {
921
- return validateWebsiteUrlWithError(rawUrl, ConfigError, options);
922
- }
923
- export function validateWebsiteInputUrl(rawUrl, options) {
924
- return validateWebsiteUrlWithError(rawUrl, UsageError, options);
925
- }
926
- function validateWebsiteUrlWithError(rawUrl, ErrorType, options) {
927
- if (!rawUrl) {
928
- throw new ErrorType("Website provider requires a URL");
929
- }
930
- let parsed;
931
- try {
932
- parsed = new URL(rawUrl);
933
- }
934
- catch {
935
- throw new ErrorType(`Website URL is not valid: "${rawUrl}"`);
936
- }
937
- if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
938
- throw new ErrorType(`Website URL must use http:// or https://, got "${parsed.protocol}" in "${rawUrl}"`);
939
- }
940
- if (parsed.username || parsed.password) {
941
- throw new ErrorType("Website URL must not contain embedded credentials");
942
- }
943
- assertWebsiteRequestUrl(parsed.toString(), ErrorType, options);
944
- parsed.hash = "";
945
- return normalizeSiteUrl(parsed.toString());
946
- }
947
- function normalizeSiteUrl(rawUrl) {
948
- const parsed = new URL(rawUrl);
949
- parsed.hash = "";
950
- if (parsed.pathname !== "/" && parsed.pathname.endsWith("/")) {
951
- parsed.pathname = parsed.pathname.replace(/\/+$/, "");
952
- }
953
- return parsed.toString();
954
- }
955
873
  function normalizeCrawlUrl(rawUrl) {
956
874
  try {
957
875
  const parsed = new URL(rawUrl);
@@ -1024,15 +942,12 @@ function uniqueSlug(base, used) {
1024
942
  used.add(candidate);
1025
943
  return candidate;
1026
944
  }
1027
- function coercePositiveInt(value, fallback) {
945
+ export function resolvePositiveInt(value, fallback, optionName) {
946
+ if (value === undefined)
947
+ return fallback;
1028
948
  if (typeof value === "number" && Number.isInteger(value) && value > 0)
1029
949
  return value;
1030
- if (typeof value === "string") {
1031
- const parsed = Number.parseInt(value, 10);
1032
- if (Number.isInteger(parsed) && parsed > 0)
1033
- return parsed;
1034
- }
1035
- return fallback;
950
+ throw new ConfigError(`Invalid value for ${optionName}: expected a positive integer, got ${JSON.stringify(value)}.`);
1036
951
  }
1037
952
  function looksLikeMarkup(body) {
1038
953
  return /<html[\s>]|<body[\s>]|<\/[a-z][\w:-]*>/i.test(body);
@@ -1101,5 +1016,6 @@ function safeCodePointToString(value) {
1101
1016
  return undefined;
1102
1017
  }
1103
1018
  }
1019
+ export { getWebsiteCachePaths, normalizeSiteUrl, shouldAllowPrivateWebsiteHostsForTests, shouldAllowPrivateWebsiteUrlForTests, validateWebsiteInputUrl, validateWebsiteUrl, } from "../website-url.js";
1104
1020
  // Re-exported for existing importers (the SSRF suite pins these entry points).
1105
1021
  export { assertResolvedHostAllowed } from "./host-guard.js";
@@ -0,0 +1,73 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Leaf URL and cache-path derivations shared by website source providers and
6
+ * the website ingest pipeline.
7
+ *
8
+ * Keep this module independent of website-ingest and the snapshot-fetcher
9
+ * registry. Provider registration must be safe to import from low-level source
10
+ * resolution code without pulling the fetcher subgraph into that leaf.
11
+ */
12
+ import { createHash } from "node:crypto";
13
+ import path from "node:path";
14
+ import { ConfigError, UsageError } from "../core/errors.js";
15
+ import { getRegistryIndexCacheDir } from "../core/paths.js";
16
+ import { assertWebsiteRequestUrl, isLoopbackWebsiteHostname, } from "./snapshot-fetchers/host-guard.js";
17
+ export function shouldAllowPrivateWebsiteHostsForTests() {
18
+ return process.env.BUN_TEST === "1" || process.env.NODE_ENV === "test";
19
+ }
20
+ export function shouldAllowPrivateWebsiteUrlForTests(rawUrl) {
21
+ if (!shouldAllowPrivateWebsiteHostsForTests())
22
+ return false;
23
+ try {
24
+ return isLoopbackWebsiteHostname(new URL(rawUrl).hostname.toLowerCase());
25
+ }
26
+ catch {
27
+ return false;
28
+ }
29
+ }
30
+ export function getWebsiteCachePaths(siteUrl, cacheRootOverride) {
31
+ const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
32
+ const rootDir = path.join(cacheRootOverride ?? getRegistryIndexCacheDir(), `website-${key}`);
33
+ return {
34
+ rootDir,
35
+ stashDir: path.join(rootDir, "stash"),
36
+ manifestPath: path.join(rootDir, "manifest.json"),
37
+ };
38
+ }
39
+ export function validateWebsiteUrl(rawUrl, options) {
40
+ return validateWebsiteUrlWithError(rawUrl, ConfigError, options);
41
+ }
42
+ export function validateWebsiteInputUrl(rawUrl, options) {
43
+ return validateWebsiteUrlWithError(rawUrl, UsageError, options);
44
+ }
45
+ function validateWebsiteUrlWithError(rawUrl, ErrorType, options) {
46
+ if (!rawUrl) {
47
+ throw new ErrorType("Website provider requires a URL");
48
+ }
49
+ let parsed;
50
+ try {
51
+ parsed = new URL(rawUrl);
52
+ }
53
+ catch {
54
+ throw new ErrorType(`Website URL is not valid: "${rawUrl}"`);
55
+ }
56
+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
57
+ throw new ErrorType(`Website URL must use http:// or https://, got "${parsed.protocol}" in "${rawUrl}"`);
58
+ }
59
+ if (parsed.username || parsed.password) {
60
+ throw new ErrorType("Website URL must not contain embedded credentials");
61
+ }
62
+ assertWebsiteRequestUrl(parsed.toString(), ErrorType, options);
63
+ parsed.hash = "";
64
+ return normalizeSiteUrl(parsed.toString());
65
+ }
66
+ export function normalizeSiteUrl(rawUrl) {
67
+ const parsed = new URL(rawUrl);
68
+ parsed.hash = "";
69
+ if (parsed.pathname !== "/" && parsed.pathname.endsWith("/")) {
70
+ parsed.pathname = parsed.pathname.replace(/\/+$/, "");
71
+ }
72
+ return parsed.toString();
73
+ }
@@ -9,12 +9,12 @@ export function assertMigrationRegistry(migrations) {
9
9
  seen.add(migration.id);
10
10
  }
11
11
  }
12
- function migrationsTableExists(db) {
12
+ export function migrationLedgerExists(db) {
13
13
  return !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'schema_migrations'").get();
14
14
  }
15
15
  /** Inspect the database's applied IDs against the exact ordered registry prefix. */
16
16
  function inspectLedgerAgainst(db, registryIds) {
17
- if (!migrationsTableExists(db))
17
+ if (!migrationLedgerExists(db))
18
18
  return { status: registryIds.length === 0 ? "current" : "old", migrationIds: [] };
19
19
  const rows = db.prepare("SELECT id FROM schema_migrations ORDER BY rowid").all();
20
20
  const migrationIds = rows.map((row) => row.id);
@@ -53,13 +53,6 @@ export function assertMigrationLedger(db, migrations) {
53
53
  }
54
54
  return state;
55
55
  }
56
- export function assertCurrentMigrationLedger(db, migrations) {
57
- const state = assertMigrationLedger(db, migrations);
58
- if (state.status !== "current") {
59
- throw new Error(`Refusing to open an obsolete writable schema; run \`akm migrate apply\`: ${state.detail ?? "pending migrations"}.`);
60
- }
61
- return state;
62
- }
63
56
  /**
64
57
  * Create the migrations ledger table if it does not exist. Must be called
65
58
  * unconditionally on every open so a fresh database bootstraps correctly.
@@ -73,13 +66,14 @@ export function ensureMigrationsTable(db) {
73
66
  `);
74
67
  }
75
68
  /**
76
- * Apply every pending migration, one transaction per migration.
69
+ * Apply every pending migration, normally one transaction per migration.
77
70
  *
78
71
  * Each migration is applied in its own transaction so a failure in migration N
79
72
  * does not roll back already-applied migrations 1..N-1. The migration row is
80
- * inserted AFTER the DDL succeeds, so a crash mid-migration leaves no row and
81
- * the migration is retried on next open (all DDL in `up` uses IF NOT EXISTS so
82
- * the retry is safe).
73
+ * inserted after the DDL succeeds in the same transaction, so a crash rolls
74
+ * back both that migration's SQL and its ledger row. A caller may explicitly
75
+ * group the initial prefix when ledger initialization and dependent early
76
+ * migrations form one safety boundary.
83
77
  *
84
78
  * @param db The open SQLite database.
85
79
  * @param migrations The module's ordered, append-only migration list.
@@ -87,13 +81,44 @@ export function ensureMigrationsTable(db) {
87
81
  */
88
82
  export function runMigrations(db, migrations, opts) {
89
83
  assertMigrationRegistry(migrations);
90
- if (opts?.applyPending === false) {
84
+ const lockedPrefixThrough = opts?.lockInitialMigrationPrefixThrough;
85
+ const lockedPrefixEnd = lockedPrefixThrough
86
+ ? migrations.findIndex((migration) => migration.id === lockedPrefixThrough)
87
+ : -1;
88
+ if (lockedPrefixThrough && lockedPrefixEnd < 0) {
89
+ throw new Error(`Initial locked migration prefix ends at unknown migration ID ${lockedPrefixThrough}.`);
90
+ }
91
+ if (lockedPrefixEnd >= 0) {
92
+ withImmediateWriteLock(db, () => {
93
+ if (!migrationLedgerExists(db)) {
94
+ opts?.beforeLedgerInitializationLocked?.(db);
95
+ ensureMigrationsTable(db);
96
+ }
97
+ assertMigrationLedger(db, migrations);
98
+ for (const migration of migrations.slice(0, lockedPrefixEnd + 1)) {
99
+ const already = db.prepare("SELECT 1 FROM schema_migrations WHERE id = ?").get(migration.id);
100
+ if (already)
101
+ continue;
102
+ opts?.beforeMigration?.(migration);
103
+ opts?.beforeMigrationLocked?.(migration, db);
104
+ db.exec(migration.up);
105
+ db.prepare("INSERT INTO schema_migrations (id) VALUES (?)").run(migration.id);
106
+ }
107
+ });
91
108
  assertMigrationLedger(db, migrations);
92
- return;
93
109
  }
94
- if (migrationsTableExists(db))
110
+ else if (migrationLedgerExists(db)) {
111
+ assertMigrationLedger(db, migrations);
112
+ }
113
+ else {
114
+ withImmediateWriteLock(db, () => {
115
+ if (migrationLedgerExists(db))
116
+ return;
117
+ opts?.beforeLedgerInitializationLocked?.(db);
118
+ ensureMigrationsTable(db);
119
+ });
95
120
  assertMigrationLedger(db, migrations);
96
- ensureMigrationsTable(db);
121
+ }
97
122
  const appliedRows = db.prepare("SELECT id FROM schema_migrations ORDER BY rowid").all();
98
123
  const applied = new Set(appliedRows.map((r) => r.id));
99
124
  for (const migration of migrations) {
@@ -103,12 +128,13 @@ export function runMigrations(db, migrations, opts) {
103
128
  withImmediateWriteLock(db, () => {
104
129
  // Re-check under the write lock. `applied` is a snapshot taken before the
105
130
  // loop, so two processes bootstrapping the same fresh DB concurrently
106
- // (both see existed=false, both run with applyPending) could each decide
131
+ // (both see existed=false) could each decide
107
132
  // to apply migration N. The first commits; the second must not re-run the
108
133
  // DDL and must not hit a UNIQUE violation on the ledger insert.
109
134
  const already = db.prepare("SELECT 1 FROM schema_migrations WHERE id = ?").get(migration.id);
110
135
  if (already)
111
136
  return;
137
+ opts?.beforeMigrationLocked?.(migration, db);
112
138
  db.exec(migration.up);
113
139
  db.prepare("INSERT INTO schema_migrations (id) VALUES (?)").run(migration.id);
114
140
  });
@@ -117,6 +143,17 @@ export function runMigrations(db, migrations, opts) {
117
143
  }
118
144
  /** Attempts to acquire the write lock before giving up to the caller. */
119
145
  const IMMEDIATE_LOCK_MAX_ATTEMPTS = 5;
146
+ function isRetryableImmediateBeginError(error) {
147
+ const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
148
+ return (message.includes("database is locked") ||
149
+ message.includes("database table is locked") ||
150
+ message.includes("did not open a transaction"));
151
+ }
152
+ function sleepImmediateRetry(ms) {
153
+ if (ms <= 0)
154
+ return;
155
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
156
+ }
120
157
  /**
121
158
  * Run `fn` inside a `BEGIN IMMEDIATE` transaction.
122
159
  *
@@ -139,24 +176,42 @@ function withImmediateWriteLock(db, fn) {
139
176
  for (let attempt = 1; attempt <= IMMEDIATE_LOCK_MAX_ATTEMPTS; attempt++) {
140
177
  try {
141
178
  db.exec("BEGIN IMMEDIATE");
179
+ if (!db.inTransaction) {
180
+ throw new Error("BEGIN IMMEDIATE did not open a transaction (phantom contention state)");
181
+ }
142
182
  }
143
183
  catch (err) {
144
- // Busy despite busy_timeout (another writer holding it across the whole
145
- // window). Retry a bounded number of times before surfacing.
146
184
  lastBeginErr = err;
147
- continue;
185
+ if (isRetryableImmediateBeginError(err) && attempt < IMMEDIATE_LOCK_MAX_ATTEMPTS) {
186
+ if (db.inTransaction) {
187
+ try {
188
+ db.exec("ROLLBACK");
189
+ }
190
+ catch {
191
+ // Transaction already gone; retrying is safe because fn has not run.
192
+ }
193
+ }
194
+ sleepImmediateRetry(2 ** (attempt - 1));
195
+ continue;
196
+ }
197
+ throw err;
148
198
  }
149
199
  try {
150
200
  fn();
201
+ if (!db.inTransaction) {
202
+ throw new Error("Migration write lock invariant violated: transaction opened by BEGIN IMMEDIATE was no longer active after the migration body ran; refusing to COMMIT (writes may have escaped serialization)");
203
+ }
151
204
  db.exec("COMMIT");
152
205
  return;
153
206
  }
154
207
  catch (err) {
155
- try {
156
- db.exec("ROLLBACK");
157
- }
158
- catch {
159
- // Already rolled back by SQLite (e.g. the statement aborted the txn).
208
+ if (db.inTransaction) {
209
+ try {
210
+ db.exec("ROLLBACK");
211
+ }
212
+ catch {
213
+ // Already rolled back by SQLite (e.g. the statement aborted the txn).
214
+ }
160
215
  }
161
216
  throw err;
162
217
  }