akm-cli 0.9.0-rc.1 → 0.9.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/CHANGELOG.md +223 -9
  2. package/SECURITY.md +23 -24
  3. package/dist/assets/help/help-improve.md +10 -10
  4. package/dist/assets/hints/cli-hints-full.md +44 -10
  5. package/dist/assets/hints/cli-hints-short.md +6 -2
  6. package/dist/assets/{profiles → improve-strategies}/default.json +1 -0
  7. package/dist/assets/{profiles → improve-strategies}/graph-refresh.json +1 -1
  8. package/dist/assets/{profiles → improve-strategies}/proactive-maintenance.json +2 -3
  9. package/dist/assets/{profiles → improve-strategies}/reflect-distill.json +3 -4
  10. package/dist/assets/stash-skeleton/README.md +28 -0
  11. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +6 -0
  12. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +6 -0
  13. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +12 -1
  14. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +11 -1
  15. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +6 -0
  16. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +9 -0
  17. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +7 -0
  18. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +7 -0
  19. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +6 -0
  20. package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +98 -0
  21. package/dist/assets/stash-skeleton/facts/conventions/domains.md +63 -0
  22. package/dist/assets/stash-skeleton/facts/conventions/organization.md +127 -0
  23. package/dist/assets/tasks/core/backup.yml +1 -0
  24. package/dist/assets/tasks/core/extract.yml +1 -0
  25. package/dist/assets/tasks/core/improve.yml +1 -0
  26. package/dist/assets/tasks/core/index-refresh.yml +1 -0
  27. package/dist/assets/tasks/core/sync.yml +1 -0
  28. package/dist/assets/tasks/core/version-check.yml +1 -0
  29. package/dist/assets/tasks/graph-refresh-weekly.yml +4 -4
  30. package/dist/assets/templates/html/health.html +5 -1
  31. package/dist/cli/config-migrate.js +31 -138
  32. package/dist/cli/config-validate.js +10 -8
  33. package/dist/cli.js +48 -14
  34. package/dist/commands/agent/agent-dispatch.js +17 -16
  35. package/dist/commands/agent/agent-support.js +0 -24
  36. package/dist/commands/agent/contribute-cli.js +5 -15
  37. package/dist/commands/backup-cli.js +54 -0
  38. package/dist/commands/config-cli.js +45 -159
  39. package/dist/commands/env/secret.js +8 -5
  40. package/dist/commands/health/checks.js +130 -83
  41. package/dist/commands/health/html-report.js +4 -0
  42. package/dist/commands/health/improve-metrics.js +30 -32
  43. package/dist/commands/health/llm-usage.js +19 -19
  44. package/dist/commands/health/md-report.js +4 -0
  45. package/dist/commands/health/metrics.js +2 -1
  46. package/dist/commands/health/surfaces.js +5 -4
  47. package/dist/commands/health.js +1 -1
  48. package/dist/commands/improve/consolidate/chunking.js +2 -2
  49. package/dist/commands/improve/consolidate.js +28 -25
  50. package/dist/commands/improve/distill/promote-memory.js +5 -12
  51. package/dist/commands/improve/distill/quality-gate.js +5 -7
  52. package/dist/commands/improve/distill.js +16 -5
  53. package/dist/commands/improve/eligibility.js +22 -12
  54. package/dist/commands/improve/extract-cli.js +47 -19
  55. package/dist/commands/improve/extract.js +110 -62
  56. package/dist/commands/improve/improve-cli.js +38 -16
  57. package/dist/commands/improve/improve-result-file.js +30 -24
  58. package/dist/commands/improve/improve-strategies.js +137 -0
  59. package/dist/commands/improve/improve.js +60 -30
  60. package/dist/commands/improve/locks.js +66 -45
  61. package/dist/commands/improve/loop-stages.js +75 -33
  62. package/dist/commands/improve/memory/memory-belief.js +79 -7
  63. package/dist/commands/improve/memory/memory-contradiction-detect.js +12 -4
  64. package/dist/commands/improve/preparation.js +71 -73
  65. package/dist/commands/improve/procedural.js +3 -2
  66. package/dist/commands/improve/recombine.js +2 -1
  67. package/dist/commands/improve/reflect.js +119 -214
  68. package/dist/commands/improve/shared.js +11 -5
  69. package/dist/commands/lint/base-linter.js +152 -42
  70. package/dist/commands/mv-cli.js +809 -0
  71. package/dist/commands/proposal/proposal-cli.js +18 -8
  72. package/dist/commands/proposal/propose.js +64 -69
  73. package/dist/commands/read/knowledge.js +436 -4
  74. package/dist/commands/read/remember-cli.js +39 -2
  75. package/dist/commands/read/search-cli.js +6 -1
  76. package/dist/commands/registry-cli.js +29 -14
  77. package/dist/commands/remember.js +2 -0
  78. package/dist/commands/sources/init.js +13 -14
  79. package/dist/commands/sources/schema-repair.js +2 -4
  80. package/dist/commands/sources/source-add.js +62 -73
  81. package/dist/commands/sources/source-manage.js +50 -46
  82. package/dist/commands/sources/stash-cli.js +41 -4
  83. package/dist/commands/tasks/default-tasks.js +12 -12
  84. package/dist/commands/tasks/tasks-cli.js +7 -3
  85. package/dist/commands/tasks/tasks.js +113 -18
  86. package/dist/commands/wiki-cli.js +9 -10
  87. package/dist/core/asset/frontmatter.js +12 -2
  88. package/dist/core/common.js +5 -3
  89. package/dist/core/config/config-io.js +28 -17
  90. package/dist/core/config/config-schema.js +358 -66
  91. package/dist/core/config/config-types.js +3 -3
  92. package/dist/core/config/config-version.js +29 -0
  93. package/dist/core/config/config-walker.js +98 -27
  94. package/dist/core/config/config.js +132 -266
  95. package/dist/core/config/deep-merge.js +41 -0
  96. package/dist/core/config/engine-semantics.js +32 -0
  97. package/dist/core/errors.js +2 -2
  98. package/dist/core/extra-params.js +61 -0
  99. package/dist/core/file-lock.js +201 -56
  100. package/dist/core/improve-result.js +178 -0
  101. package/dist/core/maintenance-barrier.js +119 -0
  102. package/dist/core/migration-backup.js +416 -0
  103. package/dist/core/paths.js +3 -0
  104. package/dist/core/redaction.js +358 -0
  105. package/dist/core/state/migrations.js +17 -2
  106. package/dist/core/state-db.js +44 -1
  107. package/dist/indexer/db/db.js +116 -1
  108. package/dist/indexer/graph/graph-extraction.js +28 -16
  109. package/dist/indexer/index-writer-lock.js +31 -24
  110. package/dist/indexer/index-written-assets.js +15 -6
  111. package/dist/indexer/indexer.js +47 -2
  112. package/dist/indexer/passes/memory-inference.js +10 -6
  113. package/dist/indexer/passes/metadata.js +250 -0
  114. package/dist/indexer/search/db-search.js +111 -44
  115. package/dist/indexer/search/fts-query.js +41 -0
  116. package/dist/indexer/search/ranking-contributors.js +48 -0
  117. package/dist/indexer/search/ranking.js +36 -23
  118. package/dist/indexer/search/search-fields.js +11 -1
  119. package/dist/integrations/agent/builder-shared.js +7 -0
  120. package/dist/integrations/agent/builders.js +5 -52
  121. package/dist/integrations/agent/config.js +3 -143
  122. package/dist/integrations/agent/detect.js +17 -2
  123. package/dist/integrations/agent/engine-resolution.js +202 -0
  124. package/dist/integrations/agent/index.js +1 -2
  125. package/dist/integrations/agent/model-aliases.js +7 -2
  126. package/dist/integrations/agent/profiles.js +6 -99
  127. package/dist/integrations/agent/runner-dispatch.js +76 -13
  128. package/dist/integrations/agent/runner.js +76 -207
  129. package/dist/integrations/agent/spawn.js +4 -6
  130. package/dist/integrations/harnesses/aider/agent-builder.js +2 -3
  131. package/dist/integrations/harnesses/aider/index.js +0 -1
  132. package/dist/integrations/harnesses/amazonq/agent-builder.js +2 -3
  133. package/dist/integrations/harnesses/amazonq/index.js +0 -1
  134. package/dist/integrations/harnesses/claude/agent-builder.js +2 -3
  135. package/dist/integrations/harnesses/claude/index.js +0 -2
  136. package/dist/integrations/harnesses/codex/agent-builder.js +2 -3
  137. package/dist/integrations/harnesses/codex/index.js +0 -1
  138. package/dist/integrations/harnesses/copilot/agent-builder.js +2 -3
  139. package/dist/integrations/harnesses/copilot/index.js +0 -1
  140. package/dist/integrations/harnesses/gemini/agent-builder.js +2 -3
  141. package/dist/integrations/harnesses/gemini/index.js +0 -1
  142. package/dist/integrations/harnesses/index.js +1 -24
  143. package/dist/integrations/harnesses/opencode/agent-builder.js +2 -3
  144. package/dist/integrations/harnesses/opencode/index.js +0 -6
  145. package/dist/integrations/harnesses/opencode-sdk/harness.js +1 -6
  146. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +274 -191
  147. package/dist/integrations/harnesses/openhands/agent-builder.js +2 -3
  148. package/dist/integrations/harnesses/openhands/index.js +0 -1
  149. package/dist/integrations/harnesses/pi/agent-builder.js +2 -3
  150. package/dist/integrations/harnesses/pi/index.js +0 -1
  151. package/dist/integrations/harnesses/types.js +1 -32
  152. package/dist/integrations/lockfile.js +32 -21
  153. package/dist/llm/client.js +48 -14
  154. package/dist/llm/feature-gate.js +15 -47
  155. package/dist/llm/graph-extract.js +1 -1
  156. package/dist/llm/index-passes.js +8 -42
  157. package/dist/llm/memory-infer-impl.js +1 -1
  158. package/dist/llm/usage-persist.js +4 -0
  159. package/dist/llm/usage-telemetry.js +35 -5
  160. package/dist/output/shapes/helpers.js +2 -1
  161. package/dist/output/shapes/passthrough.js +2 -0
  162. package/dist/output/text/helpers.js +3 -1
  163. package/dist/schemas/akm-config.json +11013 -8600
  164. package/dist/schemas/akm-task.json +87 -0
  165. package/dist/schemas/akm-workflow.json +57 -13
  166. package/dist/scripts/migrate-storage.js +8591 -509
  167. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +8623 -484
  168. package/dist/setup/detected-engines.js +142 -0
  169. package/dist/setup/engine-config.js +89 -0
  170. package/dist/setup/setup.js +236 -132
  171. package/dist/setup/steps/connection.js +61 -32
  172. package/dist/setup/steps/platforms.js +4 -4
  173. package/dist/setup/steps.js +3 -2
  174. package/dist/storage/database.js +13 -1
  175. package/dist/storage/engines/sqlite-migrations.js +1 -0
  176. package/dist/storage/repositories/improve-runs-repository.js +5 -5
  177. package/dist/storage/repositories/task-history-repository.js +78 -0
  178. package/dist/storage/repositories/workflow-runs-repository.js +9 -8
  179. package/dist/tasks/parser.js +138 -52
  180. package/dist/tasks/runner.js +71 -75
  181. package/dist/tasks/schema.js +1 -1
  182. package/dist/tasks/validator.js +11 -6
  183. package/dist/wiki/wiki.js +9 -8
  184. package/dist/workflows/authoring/workflow-program-template.yaml +3 -3
  185. package/dist/workflows/concurrency-policy.js +15 -0
  186. package/dist/workflows/db.js +65 -13
  187. package/dist/workflows/exec/brief.js +18 -24
  188. package/dist/workflows/exec/frozen-judge.js +47 -0
  189. package/dist/workflows/exec/native-executor.js +124 -65
  190. package/dist/workflows/exec/report.js +93 -33
  191. package/dist/workflows/exec/run-workflow.js +38 -25
  192. package/dist/workflows/exec/scheduler.js +12 -41
  193. package/dist/workflows/exec/step-work.js +91 -35
  194. package/dist/workflows/ir/compile.js +13 -26
  195. package/dist/workflows/ir/freeze.js +243 -0
  196. package/dist/workflows/ir/plan-hash.js +40 -5
  197. package/dist/workflows/ir/schema.js +542 -1
  198. package/dist/workflows/parser.js +7 -0
  199. package/dist/workflows/program/parser.js +132 -23
  200. package/dist/workflows/program/project.js +3 -4
  201. package/dist/workflows/program/schema.js +2 -2
  202. package/dist/workflows/resource-limits.js +20 -0
  203. package/dist/workflows/runtime/plan-classifier.js +187 -0
  204. package/dist/workflows/runtime/runs.js +54 -47
  205. package/dist/workflows/runtime/workflow-asset-loader.js +0 -22
  206. package/dist/workflows/validator.js +25 -0
  207. package/docs/data-and-telemetry.md +2 -2
  208. package/docs/migration/release-notes/0.6.0.md +1 -1
  209. package/docs/migration/release-notes/0.7.0.md +5 -4
  210. package/docs/migration/v0.8-to-v0.9.md +401 -0
  211. package/package.json +4 -2
  212. package/schemas/akm-config.json +16638 -0
  213. package/schemas/akm-task.json +87 -0
  214. package/schemas/akm-workflow.json +372 -0
  215. package/dist/commands/improve/improve-profiles.js +0 -168
  216. package/dist/core/config/config-migration.js +0 -602
  217. package/dist/core/deep-merge.js +0 -38
  218. package/dist/llm/call-ai.js +0 -62
  219. package/dist/setup/legacy-config.js +0 -106
  220. package/docs/README.md +0 -104
  221. /package/dist/assets/{profiles → improve-strategies}/catchup.json +0 -0
  222. /package/dist/assets/{profiles → improve-strategies}/consolidate.json +0 -0
  223. /package/dist/assets/{profiles → improve-strategies}/frequent.json +0 -0
  224. /package/dist/assets/{profiles → improve-strategies}/memory-focus.json +0 -0
  225. /package/dist/assets/{profiles → improve-strategies}/quick.json +0 -0
  226. /package/dist/assets/{profiles → improve-strategies}/recombine-only.json +0 -0
  227. /package/dist/assets/{profiles → improve-strategies}/synthesize.json +0 -0
  228. /package/dist/assets/{profiles → improve-strategies}/thorough.json +0 -0
@@ -0,0 +1,358 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ const ENV_PASSTHROUGH_REDACTION_POLICY = {
5
+ HOME: "path",
6
+ PATH: "path",
7
+ USER: "identifier",
8
+ LANG: "identifier",
9
+ LC_ALL: "identifier",
10
+ TERM: "identifier",
11
+ TMPDIR: "path",
12
+ SYSTEMROOT: "path",
13
+ COMSPEC: "path",
14
+ PATHEXT: "path",
15
+ WINDIR: "path",
16
+ TEMP: "path",
17
+ TMP: "path",
18
+ AKM_EVENT_SOURCE: "identifier",
19
+ OPENCODE_CONFIG: "path",
20
+ CLAUDE_CONFIG: "path",
21
+ CODEX_CONFIG: "path",
22
+ AWS_PROFILE: "identifier",
23
+ AWS_REGION: "identifier",
24
+ LLM_MODEL: "identifier",
25
+ LLM_BASE_URL: "url",
26
+ };
27
+ /** Environment names whose ordinary values identify runtime configuration rather than credentials. */
28
+ export const ENV_PASSTHROUGH_REDACTION_ALLOWLIST = new Set(Object.keys(ENV_PASSTHROUGH_REDACTION_POLICY));
29
+ const SIGNED_QUERY_KEYS = new Set([
30
+ "accesskey",
31
+ "accesskeyid",
32
+ "accesstoken",
33
+ "actortoken",
34
+ "apikey",
35
+ "assertion",
36
+ "authorization",
37
+ "authorizationcode",
38
+ "authreqid",
39
+ "clientassertion",
40
+ "clientsecret",
41
+ "code",
42
+ "codeverifier",
43
+ "credential",
44
+ "devicecode",
45
+ "googleaccessid",
46
+ "idtoken",
47
+ "idtokenhint",
48
+ "initialaccesstoken",
49
+ "key",
50
+ "loginhinttoken",
51
+ "logouthint",
52
+ "logouttoken",
53
+ "nonce",
54
+ "oauthcode",
55
+ "oauthtoken",
56
+ "oauthverifier",
57
+ "password",
58
+ "refreshtoken",
59
+ "registrationaccesstoken",
60
+ "requesturi",
61
+ "response",
62
+ "secret",
63
+ "sessiontoken",
64
+ "sharedaccesssignature",
65
+ "sig",
66
+ "signature",
67
+ "softwarestatement",
68
+ "state",
69
+ "subjecttoken",
70
+ "token",
71
+ "usercode",
72
+ "verifier",
73
+ "xamzcredential",
74
+ "xamzsecuritytoken",
75
+ "xamzsignature",
76
+ "xgoogcredential",
77
+ "xgoogsignature",
78
+ ]);
79
+ function normalizedQueryKey(key) {
80
+ return key.toLowerCase().replaceAll(/[^a-z0-9]/g, "");
81
+ }
82
+ function collectCredentialParameters(params, values) {
83
+ let found = false;
84
+ for (const [key, value] of params) {
85
+ if (!SIGNED_QUERY_KEYS.has(normalizedQueryKey(key)))
86
+ continue;
87
+ found = true;
88
+ if (value)
89
+ values?.add(value);
90
+ }
91
+ return found;
92
+ }
93
+ function addEncodedCredentialVariants(rawValue, values) {
94
+ if (!rawValue || !values)
95
+ return;
96
+ values.add(rawValue);
97
+ let decoded;
98
+ try {
99
+ decoded = decodeURIComponent(rawValue.replaceAll("+", " "));
100
+ }
101
+ catch {
102
+ return;
103
+ }
104
+ if (!decoded)
105
+ return;
106
+ values.add(decoded);
107
+ const encoded = encodeURIComponent(decoded);
108
+ values.add(encoded);
109
+ values.add(encoded.replaceAll("%20", "+"));
110
+ values.add(encoded.replace(/%[0-9A-F]{2}/g, (sequence) => sequence.toLowerCase()));
111
+ }
112
+ function collectEncodedCredentialParameters(raw, values) {
113
+ let found = false;
114
+ for (const part of raw.split("&")) {
115
+ const separator = part.indexOf("=");
116
+ if (separator < 0)
117
+ continue;
118
+ const rawKey = part.slice(0, separator);
119
+ const rawValue = part.slice(separator + 1);
120
+ let key = rawKey;
121
+ try {
122
+ key = decodeURIComponent(rawKey.replaceAll("+", " "));
123
+ }
124
+ catch {
125
+ // Keep the raw key so malformed credential-shaped input still fails closed.
126
+ }
127
+ if (!SIGNED_QUERY_KEYS.has(normalizedQueryKey(key)))
128
+ continue;
129
+ found = true;
130
+ addEncodedCredentialVariants(rawValue, values);
131
+ }
132
+ return found;
133
+ }
134
+ function collectCredentialFragment(fragment, values) {
135
+ const directDecoded = fragment.includes("=") && collectCredentialParameters(new URLSearchParams(fragment), values);
136
+ const directEncoded = fragment.includes("=") && collectEncodedCredentialParameters(fragment, values);
137
+ let found = directDecoded || directEncoded;
138
+ const nestedQuery = fragment.indexOf("?");
139
+ if (nestedQuery >= 0) {
140
+ const query = fragment.slice(nestedQuery + 1);
141
+ const decoded = collectCredentialParameters(new URLSearchParams(query), values);
142
+ const encoded = collectEncodedCredentialParameters(query, values);
143
+ found = decoded || encoded || found;
144
+ }
145
+ return found;
146
+ }
147
+ function inspectCredentialBearingUrl(value, values) {
148
+ const trimmed = value.trim();
149
+ if (!trimmed)
150
+ return false;
151
+ try {
152
+ const url = new URL(trimmed);
153
+ let found = false;
154
+ for (const userInfo of [url.username, url.password]) {
155
+ if (!userInfo)
156
+ continue;
157
+ found = true;
158
+ values?.add(userInfo);
159
+ try {
160
+ values?.add(decodeURIComponent(userInfo));
161
+ }
162
+ catch {
163
+ values?.add(userInfo);
164
+ }
165
+ }
166
+ const decodedQuery = collectCredentialParameters(url.searchParams, values);
167
+ const encodedQuery = collectEncodedCredentialParameters(url.search.slice(1), values);
168
+ found = decodedQuery || encodedQuery || found;
169
+ return collectCredentialFragment(url.hash.slice(1), values) || found;
170
+ }
171
+ catch {
172
+ // Fail closed for malformed URL-like values carrying the same credential shapes.
173
+ let found = /^[a-z][a-z0-9+.-]*:\/\/[^/\s?#]*@/i.test(trimmed);
174
+ const query = trimmed.indexOf("?");
175
+ const fragment = trimmed.indexOf("#");
176
+ if (query >= 0) {
177
+ const queryText = trimmed.slice(query + 1, fragment >= 0 ? fragment : undefined);
178
+ const decodedQuery = collectCredentialParameters(new URLSearchParams(queryText), values);
179
+ const encodedQuery = collectEncodedCredentialParameters(queryText, values);
180
+ found = decodedQuery || encodedQuery || found;
181
+ }
182
+ if (fragment >= 0) {
183
+ const fragmentText = trimmed.slice(fragment + 1);
184
+ found = collectCredentialFragment(fragmentText, values) || found;
185
+ }
186
+ return found;
187
+ }
188
+ }
189
+ /** Collect exact secrets plus decoded values embedded in credential-bearing URLs. */
190
+ export function collectSensitiveValues(rawValues) {
191
+ const values = new Set();
192
+ for (const value of rawValues) {
193
+ if (value === undefined || value.length === 0)
194
+ continue;
195
+ values.add(value);
196
+ const trimmed = value.trim();
197
+ if (inspectCredentialBearingUrl(value, values) && trimmed)
198
+ values.add(trimmed);
199
+ }
200
+ return [...values];
201
+ }
202
+ /**
203
+ * Decide whether an allowlisted passthrough value may cross an output boundary.
204
+ * The name allowlist never overrides value inspection: URL userinfo and signed
205
+ * query credentials remain secret even under ordinarily non-secret names.
206
+ */
207
+ export function isEnvPassthroughValueSafeToExpose(name, value) {
208
+ if (value === undefined)
209
+ return true;
210
+ const policy = ENV_PASSTHROUGH_REDACTION_POLICY[name];
211
+ if (!policy)
212
+ return false;
213
+ const classifiedPolicy = policy;
214
+ switch (classifiedPolicy) {
215
+ case "identifier":
216
+ case "path":
217
+ case "url":
218
+ return !inspectCredentialBearingUrl(value);
219
+ default: {
220
+ const exhaustive = classifiedPolicy;
221
+ return exhaustive;
222
+ }
223
+ }
224
+ }
225
+ function utf8SequenceLength(firstByte) {
226
+ if (firstByte <= 0x7f)
227
+ return 1;
228
+ if (firstByte >= 0xc2 && firstByte <= 0xdf)
229
+ return 2;
230
+ if (firstByte >= 0xe0 && firstByte <= 0xef)
231
+ return 3;
232
+ if (firstByte >= 0xf0 && firstByte <= 0xf4)
233
+ return 4;
234
+ return 0;
235
+ }
236
+ function normalizeEncodedText(value, plusAsSpace) {
237
+ let text = "";
238
+ const starts = new Int32Array(value.length);
239
+ let mappingLength = 0;
240
+ const append = (decoded, start) => {
241
+ text += decoded;
242
+ for (let index = 0; index < decoded.length; index++) {
243
+ starts[mappingLength] = start;
244
+ mappingLength++;
245
+ }
246
+ };
247
+ for (let index = 0; index < value.length;) {
248
+ if (value[index] === "%" && /^[0-9a-f]{2}$/i.test(value.slice(index + 1, index + 3))) {
249
+ const start = index;
250
+ const firstByte = Number.parseInt(value.slice(index + 1, index + 3), 16);
251
+ const sequenceLength = utf8SequenceLength(firstByte);
252
+ if (sequenceLength > 0) {
253
+ const bytes = [firstByte];
254
+ let cursor = index + 3;
255
+ while (bytes.length < sequenceLength &&
256
+ value[cursor] === "%" &&
257
+ /^[0-9a-f]{2}$/i.test(value.slice(cursor + 1, cursor + 3))) {
258
+ bytes.push(Number.parseInt(value.slice(cursor + 1, cursor + 3), 16));
259
+ cursor += 3;
260
+ }
261
+ if (bytes.length === sequenceLength) {
262
+ try {
263
+ const decoded = new TextDecoder("utf-8", { fatal: true }).decode(Uint8Array.from(bytes));
264
+ index = cursor;
265
+ append(decoded, start);
266
+ continue;
267
+ }
268
+ catch {
269
+ // Treat invalid UTF-8 percent sequences as literal text.
270
+ }
271
+ }
272
+ }
273
+ }
274
+ const start = index;
275
+ const codePoint = value.codePointAt(index);
276
+ if (codePoint === undefined)
277
+ break;
278
+ const character = String.fromCodePoint(codePoint);
279
+ index += character.length;
280
+ append(plusAsSpace && character === "+" ? " " : character, start);
281
+ }
282
+ return { text, starts, sourceLength: value.length };
283
+ }
284
+ function addMappedMatches(coverageDelta, haystack, needle) {
285
+ if (!needle)
286
+ return;
287
+ let offset = 0;
288
+ while (offset <= haystack.text.length - needle.length) {
289
+ const match = haystack.text.indexOf(needle, offset);
290
+ if (match < 0)
291
+ break;
292
+ const start = haystack.starts[match];
293
+ const normalizedEnd = match + needle.length;
294
+ const end = normalizedEnd < haystack.text.length ? haystack.starts[normalizedEnd] : haystack.sourceLength;
295
+ coverageDelta[start]++;
296
+ coverageDelta[end]--;
297
+ offset = match + Math.max(needle.length, 1);
298
+ }
299
+ }
300
+ /**
301
+ * Replace exact sensitive values in text. Longer values are replaced first so
302
+ * an overlapping prefix cannot expose the suffix of a longer credential.
303
+ */
304
+ export function redactSensitiveText(text, sensitiveValues) {
305
+ const values = [...new Set(sensitiveValues)]
306
+ .filter((value) => value.length > 0)
307
+ .sort((a, b) => b.length - a.length || a.localeCompare(b));
308
+ if (values.length === 0)
309
+ return text;
310
+ if (!text.includes("%") && !text.includes("+")) {
311
+ let redacted = text;
312
+ for (const value of values)
313
+ redacted = redacted.replaceAll(value, "[REDACTED]");
314
+ return redacted;
315
+ }
316
+ const coverageDelta = new Int32Array(text.length + 1);
317
+ const addMatchesForMode = (plusAsSpace) => {
318
+ const haystack = normalizeEncodedText(text, plusAsSpace);
319
+ for (const value of values) {
320
+ addMappedMatches(coverageDelta, haystack, normalizeEncodedText(value, plusAsSpace).text);
321
+ }
322
+ };
323
+ addMatchesForMode(false);
324
+ if (text.includes("+"))
325
+ addMatchesForMode(true);
326
+ let redacted = "";
327
+ let coverage = 0;
328
+ let offset = 0;
329
+ let found = false;
330
+ for (let index = 0; index <= text.length; index++) {
331
+ const wasCovered = coverage > 0;
332
+ coverage += coverageDelta[index];
333
+ const isCovered = coverage > 0;
334
+ if (!wasCovered && isCovered) {
335
+ redacted += `${text.slice(offset, index)}[REDACTED]`;
336
+ found = true;
337
+ }
338
+ else if (wasCovered && !isCovered) {
339
+ offset = index;
340
+ }
341
+ }
342
+ return found ? redacted + text.slice(offset) : text;
343
+ }
344
+ /** Recursively redact string leaves before a structured value crosses a durable/output boundary. */
345
+ export function redactSensitiveValue(value, sensitiveValues) {
346
+ const values = [...sensitiveValues];
347
+ const redact = (entry) => {
348
+ if (typeof entry === "string")
349
+ return redactSensitiveText(entry, values);
350
+ if (Array.isArray(entry))
351
+ return entry.map(redact);
352
+ if (entry && typeof entry === "object") {
353
+ return Object.fromEntries(Object.entries(entry).map(([key, child]) => [redactSensitiveText(key, values), redact(child)]));
354
+ }
355
+ return entry;
356
+ };
357
+ return redact(value);
358
+ }
@@ -2,6 +2,7 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import { runMigrations as runSqliteMigrations } from "../../storage/engines/sqlite-migrations.js";
5
+ import { ensureMigrationBackup } from "../migration-backup.js";
5
6
  const MIGRATIONS = [
6
7
  // ── Migration 001 — initial schema ──────────────────────────────────────────
7
8
  {
@@ -756,6 +757,15 @@ const MIGRATIONS = [
756
757
  ON improve_cycle_metrics(ts);
757
758
  `,
758
759
  },
760
+ // Keep the historical profile column untouched. New 0.9 runs identify the
761
+ // selected improve strategy in this additive column.
762
+ {
763
+ id: "017-improve-run-strategy",
764
+ up: `
765
+ ALTER TABLE improve_runs ADD COLUMN strategy TEXT;
766
+ CREATE INDEX IF NOT EXISTS idx_improve_runs_strategy_started ON improve_runs(strategy, started_at);
767
+ `,
768
+ },
759
769
  ];
760
770
  /**
761
771
  * Apply every pending migration in a single transaction per migration.
@@ -765,6 +775,11 @@ const MIGRATIONS = [
765
775
  *
766
776
  * Called automatically by `openStateDatabase()`.
767
777
  */
768
- export function runMigrations(db) {
769
- runSqliteMigrations(db, MIGRATIONS);
778
+ export function runMigrations(db, options) {
779
+ runSqliteMigrations(db, MIGRATIONS, {
780
+ beforeMigration(migration) {
781
+ if (options?.ensureCutoverBackup && migration.id === "017-improve-run-strategy")
782
+ ensureMigrationBackup();
783
+ },
784
+ });
770
785
  }
@@ -55,8 +55,11 @@
55
55
  *
56
56
  * @module state-db
57
57
  */
58
+ import fs from "node:fs";
58
59
  import path from "node:path";
59
60
  import { openManagedDatabase, withManagedDb, withManagedDbAsync } from "../storage/managed-db.js";
61
+ import { acquireMaintenanceActivitySync } from "./maintenance-barrier.js";
62
+ import { ensureMigrationBackup, getMigrationBackupDir } from "./migration-backup.js";
60
63
  import { getDataDir } from "./paths.js";
61
64
  // ── Path helper ──────────────────────────────────────────────────────────────
62
65
  /**
@@ -96,7 +99,47 @@ export function getStateDbPath() {
96
99
  * narrow when a post-inference reindex overlapped a parallel event write.
97
100
  */
98
101
  export function openStateDatabase(dbPath) {
99
- return openManagedDatabase({ path: dbPath ?? getStateDbPath(), init: runMigrations });
102
+ const canonicalPath = getStateDbPath();
103
+ const resolvedPath = dbPath ?? canonicalPath;
104
+ const isCanonical = path.resolve(resolvedPath) === path.resolve(canonicalPath);
105
+ const releaseActivity = isCanonical ? acquireMaintenanceActivitySync("state-db") : undefined;
106
+ try {
107
+ // This must precede mkdir/open: opening an absent SQLite path creates it and
108
+ // would make the recovery manifest falsely record a fresh DB as pre-existing.
109
+ if (isCanonical && !fs.existsSync(getMigrationBackupDir()))
110
+ ensureMigrationBackup();
111
+ const db = openManagedDatabase({
112
+ path: resolvedPath,
113
+ init: (db) => runMigrations(db, { ensureCutoverBackup: isCanonical }),
114
+ });
115
+ if (!releaseActivity)
116
+ return db;
117
+ let closed = false;
118
+ return {
119
+ prepare: db.prepare.bind(db),
120
+ exec: db.exec.bind(db),
121
+ run: db.run.bind(db),
122
+ transaction: db.transaction.bind(db),
123
+ get inTransaction() {
124
+ return db.inTransaction;
125
+ },
126
+ close() {
127
+ if (closed)
128
+ return;
129
+ closed = true;
130
+ try {
131
+ db.close();
132
+ }
133
+ finally {
134
+ releaseActivity();
135
+ }
136
+ },
137
+ };
138
+ }
139
+ catch (error) {
140
+ releaseActivity?.();
141
+ throw error;
142
+ }
100
143
  }
101
144
  /**
102
145
  * Run `fn` against state.db, owning the handle unless one is borrowed. The loan
@@ -1,6 +1,7 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import fs from "node:fs";
4
5
  import { createRequire } from "node:module";
5
6
  import path from "node:path";
6
7
  import { parseAssetRef } from "../../core/asset/asset-ref.js";
@@ -9,10 +10,11 @@ import { getDbPath } from "../../core/paths.js";
9
10
  import { warn } from "../../core/warn.js";
10
11
  import { cosineSimilarity } from "../../llm/embedders/types.js";
11
12
  import { sha256Hex } from "../../runtime.js";
13
+ import { openDatabase } from "../../storage/database.js";
12
14
  import { openManagedDatabase } from "../../storage/managed-db.js";
13
15
  import { computeNextUtility, HIGH_UTILITY_THRESHOLD, UTILITY_REVIEW_THRESHOLD, } from "../feedback/utility-policy.js";
14
16
  import { buildPrefixQuery, sanitizeFtsQuery } from "../search/fts-query.js";
15
- import { buildSearchFields } from "../search/search-fields.js";
17
+ import { buildSearchFields, buildSearchText } from "../search/search-fields.js";
16
18
  import { ENTRY_COLUMNS, rowToIndexedEntry } from "./entry-mapper.js";
17
19
  import { ensureSchema } from "./schema.js";
18
20
  export { HIGH_UTILITY_THRESHOLD, sanitizeFtsQuery, UTILITY_REVIEW_THRESHOLD };
@@ -64,6 +66,16 @@ export function openExistingDatabase(dbPath) {
64
66
  // so init only loads the vec extension, it does not run ensureSchema.
65
67
  return openManagedDatabase({ path: dbPath ?? getDbPath(), init: loadVecExtension });
66
68
  }
69
+ /**
70
+ * Open an existing index for queries without creating directories, a database
71
+ * file, journals, or running write-capable pragmas/schema initialization.
72
+ */
73
+ export function openReadonlyExistingDatabase(dbPath) {
74
+ const resolvedPath = dbPath ?? getDbPath();
75
+ if (!fs.existsSync(resolvedPath))
76
+ return undefined;
77
+ return openDatabase(resolvedPath, { readonly: true, create: false });
78
+ }
67
79
  export function closeDatabase(db) {
68
80
  db.close();
69
81
  }
@@ -301,6 +313,109 @@ export function getBaseBeliefStatesForDerivedTwins(db, twinIds) {
301
313
  }
302
314
  return out;
303
315
  }
316
+ /**
317
+ * SPEC-7 (`akm mv`): re-key an entries row IN PLACE after an on-disk rename.
318
+ *
319
+ * The row id is preserved on purpose — `utility_scores`,
320
+ * `utility_scores_scoped`, and `embeddings` are keyed by `entry_id`, so an
321
+ * UPDATE (rather than a delete + insert under the new `entry_key`) is what
322
+ * keeps the asset's accumulated usage-ranking history attached across a
323
+ * rename. (`asset_salience` / `asset_outcome` live in state.db keyed by
324
+ * `asset_ref` TEXT and are re-keyed separately by `akm mv` — see
325
+ * mv-cli.ts `rekeyStateDbForMove`.) `entry_json.name` (and `filename`, when
326
+ * present) is patched and `search_text` rebuilt so search reflects the new
327
+ * name; the row is marked FTS-dirty for the caller's
328
+ * `rebuildFts({incremental: true})`.
329
+ *
330
+ * `usage_events.entry_ref` rows for the old ref are rewritten to the new ref
331
+ * in the same transaction — both the bare `type:name` spelling and the
332
+ * origin-qualified `origin//type:name` spelling (search/show writers persist
333
+ * either, see {@link getRetrievalCounts}). Without this, events keep the old
334
+ * ref, `relinkUsageEvents` finds no matching entry after the next full
335
+ * rebuild, and the utility history the re-key exists to preserve silently
336
+ * resets. DETACHED orphan events already sitting at the new ref (entry_id
337
+ * NULL — a deleted stranger's history) are deleted first, so the moved asset
338
+ * never adopts them (live asset's history wins, matching the stale-row
339
+ * eviction below).
340
+ *
341
+ * A stale row already occupying `newEntryKey` (the caller has verified no
342
+ * FILE exists at the target, so such a row can only be a leftover for a
343
+ * deleted file) is evicted first — through {@link deleteRelatedRows}, so its
344
+ * child rows (embeddings, entries_vec, utility scores, usage events) go with
345
+ * it. A bare `DELETE FROM entries` would trip the non-CASCADE `embeddings`
346
+ * FK under `PRAGMA foreign_keys = ON` and roll back the whole re-key.
347
+ * The moved row keeps its id.
348
+ *
349
+ * Returns the surviving row id, or `null` when no row matches `oldEntryKey`
350
+ * (nothing indexed under the old name — the caller falls open and the next
351
+ * full `akm index` picks the file up as a fresh entry).
352
+ */
353
+ export function rekeyEntryInPlace(db, opts) {
354
+ const row = db.prepare("SELECT id, entry_json, search_text FROM entries WHERE entry_key = ?").get(opts.oldEntryKey);
355
+ if (!row)
356
+ return null;
357
+ // Patch the JSON payload. On corrupt entry_json still re-key key + paths so
358
+ // the utility history survives; the next full index heals the JSON.
359
+ let entryJson = row.entry_json;
360
+ let searchText = row.search_text;
361
+ try {
362
+ const entry = JSON.parse(row.entry_json);
363
+ entry.name = opts.newName;
364
+ if (typeof entry.filename === "string")
365
+ entry.filename = path.basename(opts.newFilePath);
366
+ if (opts.newDerivedFrom !== undefined)
367
+ entry.derivedFrom = opts.newDerivedFrom;
368
+ entryJson = JSON.stringify(entry);
369
+ searchText = buildSearchText(entry);
370
+ }
371
+ catch {
372
+ /* corrupt entry_json — key/path-only re-key */
373
+ }
374
+ db.transaction(() => {
375
+ const stale = db.prepare("SELECT id FROM entries WHERE entry_key = ?").get(opts.newEntryKey);
376
+ if (stale && stale.id !== row.id) {
377
+ // Full child-row cleanup (embeddings, entries_vec, utility scores,
378
+ // usage events, FTS + dirty marks) BEFORE the entries delete: the
379
+ // `embeddings` FK is non-CASCADE and `foreign_keys = ON`, so a bare
380
+ // entries delete would throw and roll back the entire re-key; and
381
+ // without it the FK-less child rows would orphan permanently.
382
+ deleteRelatedRows(db, [{ id: stale.id }]);
383
+ db.prepare("DELETE FROM entries WHERE id = ?").run(stale.id);
384
+ }
385
+ db.prepare("UPDATE entries SET entry_key = ?, dir_path = ?, file_path = ?, entry_json = ?, search_text = ? WHERE id = ?").run(opts.newEntryKey, path.dirname(opts.newFilePath), opts.newFilePath, entryJson, searchText, row.id);
386
+ if (opts.newDerivedFrom !== undefined) {
387
+ db.prepare("UPDATE entries SET derived_from = ? WHERE id = ?").run(opts.newDerivedFrom, row.id);
388
+ }
389
+ // Re-point usage history at the new ref (see the docstring): the bare
390
+ // spelling exactly, and the origin-qualified spelling by rewriting only
391
+ // the part after the last `//` (stored origins never contain `//`, so a
392
+ // qualified ref has exactly one — same normalization as
393
+ // getRetrievalCounts). Legacy DBs may predate usage_events.
394
+ bestEffort(() => {
395
+ // Live-asset-wins collision policy, mirroring the stale-entries eviction
396
+ // above and mv's state.db re-key: DETACHED orphan events (entry_id NULL
397
+ // — a deleted asset's history retained by a full rebuild) already
398
+ // sitting AT the new ref are evicted BEFORE the old→new rewrite. No
399
+ // stale entries row exists for them, so the deleteRelatedRows path
400
+ // never sees them (it deletes by entry_id only) — left in place, the
401
+ // moved asset would adopt the stranger's history: getRetrievalCounts
402
+ // reads by entry_ref immediately, and the next full rebuild's
403
+ // relinkUsageEvents would attach every stranger event by ref.
404
+ db.prepare("DELETE FROM usage_events WHERE entry_id IS NULL AND entry_ref = ?").run(opts.newRef);
405
+ db.prepare(`DELETE FROM usage_events
406
+ WHERE entry_id IS NULL
407
+ AND instr(entry_ref, '//') > 0
408
+ AND substr(entry_ref, instr(entry_ref, '//') + 2) = ?`).run(opts.newRef);
409
+ db.prepare("UPDATE usage_events SET entry_ref = ? WHERE entry_ref = ?").run(opts.newRef, opts.oldRef);
410
+ db.prepare(`UPDATE usage_events
411
+ SET entry_ref = substr(entry_ref, 1, instr(entry_ref, '//') + 1) || ?
412
+ WHERE instr(entry_ref, '//') > 0
413
+ AND substr(entry_ref, instr(entry_ref, '//') + 2) = ?`).run(opts.newRef, opts.oldRef);
414
+ }, "usage_events table may be missing on legacy DBs — no usage history to re-key");
415
+ db.prepare("INSERT OR IGNORE INTO entries_fts_dirty (entry_id) VALUES (?)").run(row.id);
416
+ })();
417
+ return row.id;
418
+ }
304
419
  /**
305
420
  * Phase 2A / Rec 5: bulk-load positive feedback event counts for the given
306
421
  * entry ids. Used by the utility-decay forgetting curve to stabilize