akm-cli 0.9.0-rc.13 → 0.9.0-rc.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/CHANGELOG.md +117 -23
  2. package/dist/akm-migrate +3 -1
  3. package/dist/assets/hints/cli-hints-full.md +3 -4
  4. package/dist/assets/hints/cli-hints-short.md +5 -5
  5. package/dist/assets/workflows/workflow-template.md +4 -3
  6. package/dist/cli/invocation.js +3 -2
  7. package/dist/cli/retired-commands.js +3 -0
  8. package/dist/cli/unknown-flags.js +226 -0
  9. package/dist/cli.js +19 -35
  10. package/dist/commands/agent/contribute-cli.js +15 -3
  11. package/dist/commands/feedback-cli.js +6 -3
  12. package/dist/commands/improve/collapse-detector.js +2 -3
  13. package/dist/commands/lint/base-linter.js +4 -16
  14. package/dist/commands/lint/index.js +13 -13
  15. package/dist/commands/log.js +6 -1
  16. package/dist/commands/migration-tool.js +4 -5
  17. package/dist/commands/observability-cli.js +1 -1
  18. package/dist/commands/proposal/repository.js +5 -5
  19. package/dist/commands/read/knowledge.js +2 -0
  20. package/dist/commands/registry-cli.js +5 -3
  21. package/dist/commands/sources/add-cli.js +6 -6
  22. package/dist/commands/sources/self-update.js +30 -7
  23. package/dist/commands/sources/source-add.js +17 -2
  24. package/dist/commands/tasks/tasks.js +8 -3
  25. package/dist/commands/workflow-cli.js +142 -119
  26. package/dist/core/adapter/adapters/akm-lint.js +17 -13
  27. package/dist/core/adapter/adapters/akm-task-adapter.js +14 -11
  28. package/dist/core/asset/akm-markdown.js +41 -8
  29. package/dist/core/asset/frontmatter.js +22 -0
  30. package/dist/core/asset/resolve-ref.js +23 -3
  31. package/dist/core/common.js +45 -2
  32. package/dist/core/config/config-schema.js +8 -0
  33. package/dist/core/config/schema/experimental.js +5 -13
  34. package/dist/core/config/schema/sources-bundles.js +11 -0
  35. package/dist/core/config/schema/workflow.js +3 -1
  36. package/dist/core/errors.js +5 -0
  37. package/dist/core/logs-db.js +2 -1
  38. package/dist/core/parse.js +4 -1
  39. package/dist/core/state/migrations.js +11 -14
  40. package/dist/core/state-db.js +4 -6
  41. package/dist/core/subprocess.js +6 -4
  42. package/dist/core/type-presentation.js +1 -1
  43. package/dist/indexer/indexer.js +16 -1
  44. package/dist/indexer/search/search-source.js +1 -1
  45. package/dist/indexer/walk/matchers.js +3 -1
  46. package/dist/integrations/agent/config.js +2 -2
  47. package/dist/integrations/agent/detect.js +49 -19
  48. package/dist/integrations/agent/profiles.js +10 -0
  49. package/dist/integrations/agent/spawn.js +1 -2
  50. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +7 -3
  51. package/dist/integrations/lockfile.js +11 -56
  52. package/dist/output/shapes/passthrough.js +0 -4
  53. package/dist/output/text/command-format.js +3 -3
  54. package/dist/output/text/helpers.js +1 -1
  55. package/dist/output/text/show-directives.js +4 -6
  56. package/dist/output/text/workflow-format.js +7 -62
  57. package/dist/output/text/workflow.js +1 -5
  58. package/dist/scripts/akm-migrate-node.js +58773 -0
  59. package/dist/scripts/akm-migrate.js +33976 -11391
  60. package/dist/setup/detect.js +40 -15
  61. package/dist/setup/setup.js +1 -1
  62. package/dist/sources/providers/git-stash.js +4 -2
  63. package/dist/sources/providers/website.js +5 -0
  64. package/dist/sources/snapshot-fetchers/bluesky.js +146 -0
  65. package/dist/sources/snapshot-fetchers/content-extract.js +370 -0
  66. package/dist/sources/snapshot-fetchers/fetcher-util.js +40 -0
  67. package/dist/sources/snapshot-fetchers/host-guard.js +199 -0
  68. package/dist/sources/snapshot-fetchers/registry.js +10 -1
  69. package/dist/sources/snapshot-fetchers/robots.js +348 -0
  70. package/dist/sources/snapshot-fetchers/rss.js +279 -0
  71. package/dist/sources/snapshot-fetchers/secret-seam.js +42 -0
  72. package/dist/sources/snapshot-fetchers/website-ingest.js +488 -257
  73. package/dist/sources/snapshot-fetchers/x.js +193 -0
  74. package/dist/storage/engines/sqlite-migrations.js +22 -107
  75. package/dist/tasks/runner.js +19 -16
  76. package/dist/tasks/schema.js +24 -1
  77. package/dist/workflows/exec/brief.js +1 -1
  78. package/dist/workflows/exec/frozen-judge.js +28 -2
  79. package/dist/workflows/exec/native-executor.js +18 -1
  80. package/dist/workflows/exec/report.js +11 -4
  81. package/dist/workflows/exec/run-workflow.js +103 -44
  82. package/dist/workflows/exec/step-work.js +19 -18
  83. package/dist/workflows/exec/unit-dispatch.js +4 -0
  84. package/dist/workflows/exec/workflow-engine-gate.js +11 -13
  85. package/dist/workflows/ir/compile.js +2 -2
  86. package/dist/workflows/ir/freeze.js +16 -8
  87. package/dist/workflows/ir/params.js +134 -10
  88. package/dist/workflows/ir/plan-hash.js +1 -1
  89. package/dist/workflows/ir/schema.js +6 -2
  90. package/dist/workflows/renderer.js +2 -2
  91. package/dist/workflows/runtime/checkin.js +3 -3
  92. package/dist/workflows/runtime/runs.js +50 -29
  93. package/dist/workflows/validate-summary.js +30 -14
  94. package/docs/migration/release-notes/0.9.0.md +31 -4
  95. package/docs/migration/v0.8-to-v0.9.md +69 -100
  96. package/docs/reference/data-and-telemetry.md +5 -5
  97. package/package.json +6 -2
  98. package/schemas/akm-config.json +24 -0
  99. package/schemas/akm-workflow.json +1 -1
  100. package/dist/workflows/cli.js +0 -33
@@ -0,0 +1,279 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { XMLParser } from "fast-xml-parser";
5
+ import { ResponseTooLargeError, readBodyWithByteCap } from "../../core/common.js";
6
+ import { htmlToMarkdown, isSafeLinkUrl } from "./content-extract.js";
7
+ import { fetchGuardedResponse } from "./host-guard.js";
8
+ /**
9
+ * RSS 2.0 / Atom 1.0 / RDF (RSS 1.0) feed fetcher.
10
+ *
11
+ * `matches` is deliberately loose — feed URLs are not reliably identifiable
12
+ * from their path alone — so `fetch` content-sniffs the body and returns
13
+ * `null` for anything that is not actually a feed. Returning `null` lets the
14
+ * registry fall through to the generic website crawler, which is the correct
15
+ * outcome for a `/feed`-shaped URL that serves HTML.
16
+ */
17
+ /** Feeds far larger than this are aggregators, not knowledge sources. */
18
+ const FEED_BYTE_CAP = 5 * 1024 * 1024;
19
+ const FEED_BODY_TIMEOUT_MS = 30_000;
20
+ const DEFAULT_ITEM_LIMIT = 50;
21
+ /** Paths that usually indicate a feed. Confirmed by sniffing before parsing. */
22
+ const FEED_PATH_PATTERN = /(?:\.(?:rss|atom|xml)$|\/(?:feed|rss|atom)\/?$)/i;
23
+ /**
24
+ * fast-xml-parser resolves no external entities and has no DTD/XXE surface,
25
+ * so the classic billion-laughs and file-disclosure vectors do not apply.
26
+ * The remaining risk is sheer size, which `FEED_BYTE_CAP` bounds before any
27
+ * parsing happens.
28
+ */
29
+ const xmlParser = new XMLParser({
30
+ ignoreAttributes: false,
31
+ attributeNamePrefix: "@_",
32
+ textNodeName: "#text",
33
+ isArray: (name) => ["item", "entry"].includes(name),
34
+ trimValues: true,
35
+ parseTagValue: false,
36
+ processEntities: true,
37
+ });
38
+ /** Coerce fast-xml-parser's string | {#text} | array shapes to a string. */
39
+ function text(value) {
40
+ if (typeof value === "string")
41
+ return value.trim();
42
+ if (typeof value === "number" || typeof value === "boolean")
43
+ return String(value);
44
+ if (Array.isArray(value))
45
+ return text(value[0]);
46
+ if (value && typeof value === "object") {
47
+ const node = value;
48
+ if ("#text" in node)
49
+ return text(node["#text"]);
50
+ }
51
+ return "";
52
+ }
53
+ /**
54
+ * Atom `<content>` may be type="text"/"html" (a #text body) or type="xhtml"
55
+ * (child elements). fast-xml-parser gives the latter as an object with no
56
+ * `#text`, which would otherwise read as an empty summary and silently drop
57
+ * the whole article body.
58
+ */
59
+ function atomContent(value) {
60
+ const direct = text(value);
61
+ if (direct)
62
+ return direct;
63
+ if (!value || typeof value !== "object")
64
+ return "";
65
+ const collect = (node) => {
66
+ if (typeof node === "string")
67
+ return node;
68
+ if (Array.isArray(node))
69
+ return node.map(collect).join(" ");
70
+ if (node && typeof node === "object") {
71
+ return Object.entries(node)
72
+ .filter(([key]) => !key.startsWith("@_"))
73
+ .map(([, child]) => collect(child))
74
+ .join(" ");
75
+ }
76
+ return "";
77
+ };
78
+ return collect(value);
79
+ }
80
+ /** Atom links are attribute-shaped and may repeat with different rels. */
81
+ function atomLink(value) {
82
+ const candidates = Array.isArray(value) ? value : [value];
83
+ let fallback = "";
84
+ for (const candidate of candidates) {
85
+ if (typeof candidate === "string") {
86
+ fallback ||= candidate.trim();
87
+ continue;
88
+ }
89
+ if (!candidate || typeof candidate !== "object")
90
+ continue;
91
+ const node = candidate;
92
+ const href = typeof node["@_href"] === "string" ? node["@_href"].trim() : "";
93
+ if (!href)
94
+ continue;
95
+ const rel = typeof node["@_rel"] === "string" ? node["@_rel"] : "";
96
+ if (rel === "alternate" || rel === "")
97
+ return href;
98
+ fallback ||= href;
99
+ }
100
+ return fallback;
101
+ }
102
+ /**
103
+ * Feed item bodies are attacker-controlled HTML (often inside CDATA or
104
+ * entity-encoded). Route them through the same converter the website path
105
+ * uses so dangerous-block removal and link-scheme policy apply here too — a
106
+ * plain tag-strip keeps `<script>` BODIES and leaks attribute text.
107
+ */
108
+ function htmlToPlainText(value, baseUrl) {
109
+ if (!value)
110
+ return "";
111
+ return htmlToMarkdown(value, baseUrl).replace(/\s+/g, " ").trim();
112
+ }
113
+ /** Collapse whitespace so a value cannot forge markdown structure. */
114
+ function singleLine(value) {
115
+ return value.replace(/\s+/g, " ").trim();
116
+ }
117
+ /**
118
+ * Emit a link only when it resolves to a safe http(s) URL. Relative IRIs are
119
+ * valid and common in Atom, so resolve against the feed URL before the scheme
120
+ * check rather than dropping them.
121
+ */
122
+ function safeLinkOrEmpty(value, baseUrl) {
123
+ if (!value)
124
+ return "";
125
+ try {
126
+ const resolved = new URL(value, baseUrl);
127
+ return isSafeLinkUrl(resolved) ? resolved.toString() : "";
128
+ }
129
+ catch {
130
+ return "";
131
+ }
132
+ }
133
+ function toIsoDate(value) {
134
+ if (!value)
135
+ return "";
136
+ const parsed = new Date(value);
137
+ return Number.isNaN(parsed.getTime()) ? "" : parsed.toISOString();
138
+ }
139
+ function readRssItems(channel, limit, baseUrl) {
140
+ const items = Array.isArray(channel.item) ? channel.item : [];
141
+ return items.slice(0, limit).map((raw) => {
142
+ const item = (raw ?? {});
143
+ return {
144
+ title: singleLine(text(item.title)),
145
+ link: safeLinkOrEmpty(text(item.link) || text(item.guid), baseUrl),
146
+ date: toIsoDate(text(item.pubDate) || text(item["dc:date"])),
147
+ summary: htmlToPlainText(text(item.description) || text(item["content:encoded"]), baseUrl),
148
+ };
149
+ });
150
+ }
151
+ function readAtomEntries(feed, limit, baseUrl) {
152
+ const entries = Array.isArray(feed.entry) ? feed.entry : [];
153
+ return entries.slice(0, limit).map((raw) => {
154
+ const entry = (raw ?? {});
155
+ return {
156
+ title: singleLine(text(entry.title)),
157
+ link: safeLinkOrEmpty(atomLink(entry.link), baseUrl),
158
+ date: toIsoDate(text(entry.updated) || text(entry.published)),
159
+ summary: htmlToPlainText(text(entry.summary) || atomContent(entry.content), baseUrl),
160
+ };
161
+ });
162
+ }
163
+ /** Returns null when the document is not a recognizable feed. */
164
+ export function parseFeed(xml, limit = DEFAULT_ITEM_LIMIT, baseUrl = "https://feed.invalid/") {
165
+ let doc;
166
+ try {
167
+ doc = xmlParser.parse(xml);
168
+ }
169
+ catch {
170
+ return null;
171
+ }
172
+ if (!doc || typeof doc !== "object")
173
+ return null;
174
+ const rss = doc.rss;
175
+ if (rss?.channel) {
176
+ const channel = rss.channel;
177
+ return { title: singleLine(text(channel.title)), items: readRssItems(channel, limit, baseUrl) };
178
+ }
179
+ const feed = doc.feed;
180
+ if (feed) {
181
+ return { title: singleLine(text(feed.title)), items: readAtomEntries(feed, limit, baseUrl) };
182
+ }
183
+ // RDF / RSS 1.0 hoists <item> to the document root alongside <channel>.
184
+ const rdf = doc["rdf:RDF"];
185
+ if (rdf) {
186
+ const channel = (rdf.channel ?? {});
187
+ const scope = Array.isArray(rdf.item) ? rdf : channel;
188
+ return {
189
+ title: singleLine(text(channel.title)),
190
+ items: readRssItems(scope, limit, baseUrl),
191
+ };
192
+ }
193
+ return null;
194
+ }
195
+ /** Cheap pre-parse check so HTML served at a feed-shaped URL is not parsed. */
196
+ function looksLikeFeed(body) {
197
+ const head = body.slice(0, 16_384);
198
+ return /<(?:rss\b|feed\b|rdf:RDF\b)/i.test(head);
199
+ }
200
+ function slugify(value) {
201
+ return value
202
+ .toLowerCase()
203
+ .replace(/[^a-z0-9._-]+/g, "-")
204
+ .replace(/^-+|-+$/g, "")
205
+ .slice(0, 80);
206
+ }
207
+ function preferredNameFor(url) {
208
+ const host = slugify(url.hostname);
209
+ const pathPart = slugify(url.pathname.replace(/\.(rss|atom|xml)$/i, ""));
210
+ return pathPart ? `feeds/${host}-${pathPart}` : `feeds/${host}`;
211
+ }
212
+ function renderMarkdown(feed, url) {
213
+ const sections = [];
214
+ for (const item of feed.items) {
215
+ const heading = item.title || item.link || "(untitled)";
216
+ sections.push(`## ${heading}`, "");
217
+ const meta = [];
218
+ if (item.date)
219
+ meta.push(item.date);
220
+ if (item.link)
221
+ meta.push(item.link);
222
+ if (meta.length > 0)
223
+ sections.push(meta.join(" — "), "");
224
+ if (item.summary)
225
+ sections.push(item.summary, "");
226
+ }
227
+ if (sections.length === 0)
228
+ sections.push(`No items in feed ${url.toString()}`, "");
229
+ return sections.join("\n").trimEnd();
230
+ }
231
+ const rssFetcher = {
232
+ name: "rss-feed",
233
+ matches(url) {
234
+ if (url.protocol !== "http:" && url.protocol !== "https:")
235
+ return false;
236
+ return FEED_PATH_PATTERN.test(url.pathname) || url.searchParams.has("feed");
237
+ },
238
+ async fetch(url, context) {
239
+ // Full SSRF guard chain: a feed URL is caller-supplied, and without
240
+ // manual redirect handling a public host can bounce the request into the
241
+ // private network with no revalidation.
242
+ const response = await fetchGuardedResponse(url.toString(), {
243
+ headers: {
244
+ Accept: "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8",
245
+ "User-Agent": "akm-cli rss fetcher",
246
+ },
247
+ signal: context.signal,
248
+ }, { timeoutMs: context.timeoutMs, retries: 1, allowPrivateHosts: context.allowPrivateHosts });
249
+ if (!response.ok)
250
+ return null;
251
+ let body;
252
+ try {
253
+ body = await readBodyWithByteCap(response, FEED_BYTE_CAP, {
254
+ bodyTimeoutMs: FEED_BODY_TIMEOUT_MS,
255
+ signal: context.signal,
256
+ });
257
+ }
258
+ catch (error) {
259
+ // An oversized feed is not a hard failure — fall through to the generic
260
+ // crawler rather than aborting the whole `akm bundle add`.
261
+ if (error instanceof ResponseTooLargeError)
262
+ return null;
263
+ throw error;
264
+ }
265
+ if (!looksLikeFeed(body))
266
+ return null;
267
+ const feed = parseFeed(body, DEFAULT_ITEM_LIMIT, url.toString());
268
+ if (!feed || feed.items.length === 0)
269
+ return null;
270
+ return {
271
+ url: url.toString(),
272
+ title: feed.title || url.hostname,
273
+ markdown: renderMarkdown(feed, url),
274
+ preferredName: preferredNameFor(url),
275
+ tags: ["rss", "feed", url.hostname],
276
+ };
277
+ },
278
+ };
279
+ export default rssFetcher;
@@ -0,0 +1,42 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import fs from "node:fs";
5
+ import { resolveSecretPath } from "../../core/env-secret-ref.js";
6
+ /**
7
+ * Reads a secret out of akm's secret store for the fetcher `resolveSecret`
8
+ * seam (see `FetcherContext.resolveSecret`).
9
+ *
10
+ * This module — NOT the fetchers themselves — is what imports
11
+ * `core/env-secret-ref`. That module transitively imports the source
12
+ * providers, which import the fetcher registry, so a fetcher importing it
13
+ * directly would close an import cycle. Keeping the dependency here lets the
14
+ * fetchers stay leaves while still reaching the store.
15
+ *
16
+ * Returns null for any failure — missing store, unresolvable ref, unreadable
17
+ * file. The underlying error is deliberately not surfaced: it can embed
18
+ * filesystem paths, and every failure means the same thing to a caller ("no
19
+ * secret"). The value itself is never logged.
20
+ */
21
+ export function resolveSecretFromStore(ref) {
22
+ try {
23
+ const { absPath } = resolveSecretPath(ref);
24
+ if (!fs.existsSync(absPath))
25
+ return null;
26
+ return fs.readFileSync(absPath, "utf8").trim() || null;
27
+ }
28
+ catch {
29
+ return null;
30
+ }
31
+ }
32
+ /**
33
+ * The single store-backed {@link SecretResolver}, constructed here because this
34
+ * is the only module in the sources tree sanctioned to import
35
+ * `core/env-secret-ref`. Composition roots above the import cycle
36
+ * (`indexer/indexer.ts` for the `sync()`/bundle-update path, and the
37
+ * `akm import` / `akm bundle add` command handlers) inject this so in-cycle
38
+ * consumers depend only on the `SecretResolver` interface, never on the reader.
39
+ */
40
+ export const storeSecretResolver = {
41
+ resolveSecret: resolveSecretFromStore,
42
+ };