@mulmoclaude/core 0.2.7 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/assets/skills-preset/mc-manage-automations/SKILL.md +1 -1
  2. package/dist/feeds/server/index.d.ts +1 -0
  3. package/dist/feeds/server/index.js +37 -4
  4. package/dist/feeds/server/index.js.map +1 -1
  5. package/dist/feeds/server/scheduledRefresh.d.ts +21 -0
  6. package/package.json +1 -1
  7. package/dist/collection/core/actionVisible.d.ts +0 -34
  8. package/dist/collection/core/calendarGrid.d.ts +0 -120
  9. package/dist/collection/core/deriveAll.d.ts +0 -38
  10. package/dist/collection/core/derivedFormula.d.ts +0 -18
  11. package/dist/collection/core/draft.d.ts +0 -18
  12. package/dist/collection/core/enumColors.d.ts +0 -33
  13. package/dist/collection/core/errorMessage.d.ts +0 -4
  14. package/dist/collection/core/itemLabel.d.ts +0 -12
  15. package/dist/collection/core/presentCollection.d.ts +0 -13
  16. package/dist/collection/core/promptSafety.d.ts +0 -1
  17. package/dist/collection/core/schema.d.ts +0 -382
  18. package/dist/collection/core/shortHexId.d.ts +0 -8
  19. package/dist/collection/core/sortItems.d.ts +0 -29
  20. package/dist/collection/core/uiTypes.d.ts +0 -106
  21. package/dist/collection/index.cjs +0 -794
  22. package/dist/collection/index.cjs.map +0 -1
  23. package/dist/collection/index.d.ts +0 -14
  24. package/dist/collection/paths.cjs +0 -44
  25. package/dist/collection/paths.cjs.map +0 -1
  26. package/dist/collection/server/atomic.d.ts +0 -1
  27. package/dist/collection/server/delete.d.ts +0 -38
  28. package/dist/collection/server/derive.d.ts +0 -8
  29. package/dist/collection/server/discoveredCollection.d.ts +0 -18
  30. package/dist/collection/server/discovery.d.ts +0 -239
  31. package/dist/collection/server/host.d.ts +0 -77
  32. package/dist/collection/server/index.cjs +0 -51
  33. package/dist/collection/server/io.d.ts +0 -120
  34. package/dist/collection/server/paths.d.ts +0 -52
  35. package/dist/collection/server/spawn.d.ts +0 -55
  36. package/dist/collection/server/util.d.ts +0 -3
  37. package/dist/collection/server/validate.d.ts +0 -19
  38. package/dist/collection/server/views.d.ts +0 -20
  39. package/dist/collection-watchers/config.d.ts +0 -49
  40. package/dist/collection-watchers/index.cjs +0 -553
  41. package/dist/collection-watchers/index.cjs.map +0 -1
  42. package/dist/collection-watchers/index.d.ts +0 -3
  43. package/dist/collection-watchers/reconciler.d.ts +0 -33
  44. package/dist/collection-watchers/watcher.d.ts +0 -34
  45. package/dist/deriveAll-VRWrs3SF.cjs +0 -413
  46. package/dist/deriveAll-VRWrs3SF.cjs.map +0 -1
  47. package/dist/feeds/index.cjs +0 -8
  48. package/dist/feeds/index.d.ts +0 -1
  49. package/dist/feeds/ingestTypes.d.ts +0 -57
  50. package/dist/feeds/paths.cjs +0 -48
  51. package/dist/feeds/paths.cjs.map +0 -1
  52. package/dist/feeds/paths.d.ts +0 -22
  53. package/dist/feeds/server/fetch/httpClient.d.ts +0 -8
  54. package/dist/feeds/server/fetch/rssParser.d.ts +0 -12
  55. package/dist/feeds/server/index.cjs +0 -804
  56. package/dist/feeds/server/index.cjs.map +0 -1
  57. package/dist/feeds/server/pathResolver.d.ts +0 -6
  58. package/dist/feeds/server/projectItem.d.ts +0 -8
  59. package/dist/feeds/server/refreshResult.d.ts +0 -17
  60. package/dist/feeds/server/retrievers/httpJson.d.ts +0 -3
  61. package/dist/feeds/server/retrievers/index.d.ts +0 -12
  62. package/dist/feeds/server/retrievers/registerAll.d.ts +0 -0
  63. package/dist/feeds/server/retrievers/rss.d.ts +0 -3
  64. package/dist/file-change/index.cjs +0 -72
  65. package/dist/file-change/index.cjs.map +0 -1
  66. package/dist/file-change/index.d.ts +0 -43
  67. package/dist/ingestTypes-DDOVhOYC.cjs +0 -32
  68. package/dist/ingestTypes-DDOVhOYC.cjs.map +0 -1
  69. package/dist/notifier/engine.d.ts +0 -72
  70. package/dist/notifier/index.cjs +0 -21
  71. package/dist/notifier/index.d.ts +0 -3
  72. package/dist/notifier/store.d.ts +0 -18
  73. package/dist/notifier/types.d.ts +0 -118
  74. package/dist/notifier/validate.d.ts +0 -17
  75. package/dist/notifier-bS8IEeLA.cjs +0 -577
  76. package/dist/notifier-bS8IEeLA.cjs.map +0 -1
  77. package/dist/rolldown-runtime-D6vf50IK.cjs +0 -28
  78. package/dist/scheduler/adapter.d.ts +0 -48
  79. package/dist/scheduler/index.cjs +0 -352
  80. package/dist/scheduler/index.cjs.map +0 -1
  81. package/dist/scheduler/index.d.ts +0 -2
  82. package/dist/scheduler/task-manager.d.ts +0 -51
  83. package/dist/server-Db-x7OXz.cjs +0 -1993
  84. package/dist/server-Db-x7OXz.cjs.map +0 -1
  85. package/dist/skill-bridge/index.cjs +0 -88
  86. package/dist/skill-bridge/index.cjs.map +0 -1
  87. package/dist/skill-bridge/index.d.ts +0 -30
  88. package/dist/whisper/client.cjs +0 -241
  89. package/dist/whisper/client.cjs.map +0 -1
  90. package/dist/whisper/client.d.ts +0 -35
  91. package/dist/whisper/ffmpeg.d.ts +0 -6
  92. package/dist/whisper/index.cjs +0 -433
  93. package/dist/whisper/index.cjs.map +0 -1
  94. package/dist/whisper/index.d.ts +0 -5
  95. package/dist/whisper/internal.d.ts +0 -11
  96. package/dist/whisper/models.d.ts +0 -49
  97. package/dist/whisper/sidecar.d.ts +0 -8
  98. package/dist/whisper/whisper.d.ts +0 -28
  99. package/dist/workspace-setup/assets.d.ts +0 -10
  100. package/dist/workspace-setup/index.d.ts +0 -3
  101. package/dist/workspace-setup/index.js +0 -560
  102. package/dist/workspace-setup/index.js.map +0 -1
  103. package/dist/workspace-setup/slug.d.ts +0 -6
  104. package/dist/workspace-setup/slug.js +0 -13
  105. package/dist/workspace-setup/slug.js.map +0 -1
  106. package/dist/workspace-setup/sync.d.ts +0 -94
@@ -1,804 +0,0 @@
1
- Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
- const require_rolldown_runtime = require("../../rolldown-runtime-D6vf50IK.cjs");
3
- const require_deriveAll = require("../../deriveAll-VRWrs3SF.cjs");
4
- const require_server = require("../../server-Db-x7OXz.cjs");
5
- const require_ingestTypes = require("../../ingestTypes-DDOVhOYC.cjs");
6
- const require_feeds_paths = require("../paths.cjs");
7
- const require_notifier = require("../../notifier-bS8IEeLA.cjs");
8
- let node_path = require("node:path");
9
- node_path = require_rolldown_runtime.__toESM(node_path, 1);
10
- let node_fs_promises = require("node:fs/promises");
11
- let node_crypto = require("node:crypto");
12
- let node_dns_promises = require("node:dns/promises");
13
- let node_net = require("node:net");
14
- let fast_xml_parser = require("fast-xml-parser");
15
- //#region src/feeds/server/host.ts
16
- var current = null;
17
- /** Wire the feeds engine to a host. Call once at startup, before any refresh. */
18
- function configureFeedsHost(host) {
19
- if (current && current !== host) throw new Error("@mulmoclaude/core/feeds: configureFeedsHost() was already called with a different host");
20
- current = host;
21
- }
22
- /** The configured host, or throw if `configureFeedsHost` was never called. */
23
- function requireFeedsHost() {
24
- if (!current) throw new Error("@mulmoclaude/core/feeds: configureFeedsHost() was not called by the host");
25
- return current;
26
- }
27
- /** Test-only: clear the configured host. */
28
- function resetFeedsHostForTesting() {
29
- current = null;
30
- }
31
- /** Forwarding logger so engine modules can `import { log }` without each
32
- * reaching for `requireFeedsHost().log`. */
33
- var log = {
34
- error: (prefix, msg, data) => requireFeedsHost().log.error(prefix, msg, data),
35
- warn: (prefix, msg, data) => requireFeedsHost().log.warn(prefix, msg, data),
36
- info: (prefix, msg, data) => requireFeedsHost().log.info(prefix, msg, data),
37
- debug: (prefix, msg, data) => requireFeedsHost().log.debug(prefix, msg, data)
38
- };
39
- //#endregion
40
- //#region src/feeds/server/registry.ts
41
- /** Every registered feed, as a discovered collection (carrying its
42
- * validated schema, `ingest`, and resolved `dataDir`). */
43
- async function listFeeds(workspaceRoot = requireFeedsHost().workspaceRoot) {
44
- return (await require_server.discoverCollections({ workspaceRoot })).filter((collection) => collection.source === "feed");
45
- }
46
- /** Delete a feed entirely: its records AND its `feeds/<slug>/` directory
47
- * (schema + state). Idempotent. Host-side only (backs the UI delete
48
- * button); the agent removes a feed by deleting both directories itself.
49
- *
50
- * The records dir is derived from the SLUG (`data/feeds/<slug>`), never
51
- * from the schema's `dataPath` — feeds are forced into that namespace at
52
- * discovery, so a malformed/hostile `dataPath` can't redirect this delete
53
- * at another app's data (e.g. `data/wiki`). `resolveDataDir` also rejects
54
- * any path that escapes the workspace. */
55
- async function removeFeed(workspaceRoot, slug) {
56
- const safe = require_server.safeSlugName(slug);
57
- if (safe === null) return false;
58
- const recordsDir = require_server.resolveDataDir(`data/feeds/${safe}`, workspaceRoot);
59
- try {
60
- if (recordsDir) await (0, node_fs_promises.rm)(recordsDir, {
61
- recursive: true,
62
- force: true
63
- });
64
- await (0, node_fs_promises.rm)(require_feeds_paths.feedDir(safe, workspaceRoot), {
65
- recursive: true,
66
- force: true
67
- });
68
- log.info("feeds", "feed + records removed", { slug: safe });
69
- return true;
70
- } catch (error) {
71
- log.warn("feeds", "feed remove failed", {
72
- slug: safe,
73
- error: String(error)
74
- });
75
- return false;
76
- }
77
- }
78
- //#endregion
79
- //#region src/feeds/server/retrievers/index.ts
80
- var registry = /* @__PURE__ */ new Map();
81
- function registerRetriever(kind, retriever) {
82
- registry.set(kind, retriever);
83
- }
84
- function getRetriever(kind) {
85
- return registry.get(kind);
86
- }
87
- //#endregion
88
- //#region src/feeds/server/fetch/httpClient.ts
89
- var ONE_SECOND_MS = 1e3;
90
- /** Identifies the bot to site operators. */
91
- var FEED_USER_AGENT = "MulmoClaude-FeedBot/1.0 (+https://github.com/receptron/mulmoclaude)";
92
- /** Per-request wall-clock cap so a hung server can't wedge a refresh. */
93
- var DEFAULT_FEED_TIMEOUT_MS = 30 * ONE_SECOND_MS;
94
- /** Cap on redirect hops followed (each re-checked for SSRF). */
95
- var MAX_REDIRECTS = 5;
96
- var BLOCKED_V4_CIDRS = [
97
- ["0.0.0.0", 8],
98
- ["10.0.0.0", 8],
99
- ["100.64.0.0", 10],
100
- ["127.0.0.0", 8],
101
- ["169.254.0.0", 16],
102
- ["172.16.0.0", 12],
103
- ["192.168.0.0", 16]
104
- ];
105
- function ipv4ToInt(address) {
106
- const octets = address.split(".").map(Number);
107
- if (octets.length !== 4 || octets.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return null;
108
- return (octets[0] << 24 | octets[1] << 16 | octets[2] << 8 | octets[3]) >>> 0;
109
- }
110
- function isBlockedIpv4(address) {
111
- const value = ipv4ToInt(address);
112
- if (value === null) return true;
113
- return BLOCKED_V4_CIDRS.some(([base, bits]) => {
114
- const baseInt = ipv4ToInt(base) ?? 0;
115
- const mask = bits === 0 ? 0 : 4294967295 << 32 - bits >>> 0;
116
- return (value & mask) === (baseInt & mask);
117
- });
118
- }
119
- function isBlockedIpv6(address) {
120
- const lower = address.toLowerCase();
121
- if (lower === "::1" || lower === "::") return true;
122
- if (lower.startsWith("fe80")) return true;
123
- if (lower.startsWith("fc") || lower.startsWith("fd")) return true;
124
- const mapped = /^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/.exec(lower);
125
- return mapped ? isBlockedIpv4(mapped[1]) : false;
126
- }
127
- /** True for any address we must not fetch (also blocks non-IP input). */
128
- function isBlockedIp(address) {
129
- const kind = (0, node_net.isIP)(address);
130
- if (kind === 4) return isBlockedIpv4(address);
131
- if (kind === 6) return isBlockedIpv6(address);
132
- return true;
133
- }
134
- /** Reject non-http(s) URLs and URLs that resolve to a private/loopback
135
- * address (SSRF guard). Throws with a clear reason; returns void on pass. */
136
- async function assertFetchableUrl(rawUrl) {
137
- if (!/^https?:\/\//i.test(rawUrl)) throw new Error(`refusing non-http(s) URL: ${rawUrl}`);
138
- const { hostname } = new URL(rawUrl);
139
- const host = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
140
- if ((0, node_net.isIP)(host)) {
141
- if (isBlockedIp(host)) throw new Error(`refusing to fetch a private/loopback address: ${host}`);
142
- return;
143
- }
144
- let addresses;
145
- try {
146
- addresses = await (0, node_dns_promises.lookup)(host, { all: true });
147
- } catch (err) {
148
- throw new Error(`could not resolve host '${host}': ${String(err)}`);
149
- }
150
- const blocked = addresses.find((entry) => isBlockedIp(entry.address));
151
- if (blocked) throw new Error(`refusing to fetch '${host}' — resolves to a private/loopback address (${blocked.address})`);
152
- }
153
- async function fetchOnce(url, timeoutMs) {
154
- const controller = new AbortController();
155
- const timer = setTimeout(() => controller.abort(new DOMException(`feed fetch timed out after ${timeoutMs}ms`, "TimeoutError")), timeoutMs);
156
- try {
157
- return await fetch(url, {
158
- headers: { "User-Agent": FEED_USER_AGENT },
159
- signal: controller.signal,
160
- redirect: "manual"
161
- });
162
- } finally {
163
- clearTimeout(timer);
164
- }
165
- }
166
- async function fetchGuarded(rawUrl, timeoutMs) {
167
- let current = rawUrl;
168
- for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
169
- await assertFetchableUrl(current);
170
- const response = await fetchOnce(current, timeoutMs);
171
- const redirect = response.status >= 300 && response.status < 400 && response.status !== 304 ? response.headers.get("location") : null;
172
- if (!redirect) return response;
173
- current = new URL(redirect, current).toString();
174
- }
175
- throw new Error(`too many redirects (>${MAX_REDIRECTS}) starting from ${rawUrl}`);
176
- }
177
- /** Fetch a URL as text, throwing on guard rejection, network error, or non-2xx. */
178
- async function fetchText(url, timeoutMs = DEFAULT_FEED_TIMEOUT_MS) {
179
- const response = await fetchGuarded(url, timeoutMs);
180
- if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText} fetching ${url}`);
181
- return response.text();
182
- }
183
- /** Fetch a URL as parsed JSON, throwing on guard rejection, network error, or non-2xx. */
184
- async function fetchJson(url, timeoutMs = DEFAULT_FEED_TIMEOUT_MS) {
185
- const response = await fetchGuarded(url, timeoutMs);
186
- if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText} fetching ${url}`);
187
- return response.json();
188
- }
189
- //#endregion
190
- //#region src/feeds/server/fetch/rssParser.ts
191
- function isRecord(value) {
192
- return typeof value === "object" && value !== null && !Array.isArray(value);
193
- }
194
- function isNonEmptyString(value) {
195
- return typeof value === "string" && value.trim().length > 0;
196
- }
197
- var xml = new fast_xml_parser.XMLParser({
198
- ignoreAttributes: false,
199
- attributeNamePrefix: "@_",
200
- cdataPropName: "#cdata",
201
- parseTagValue: false,
202
- parseAttributeValue: false,
203
- trimValues: true,
204
- isArray: (name) => name === "item" || name === "entry"
205
- });
206
- /** Parse an RSS/Atom/RDF feed body. Returns null when the input doesn't
207
- * look like a feed we understand. */
208
- function parseFeed(body) {
209
- const text = stripBom(body);
210
- if (!text.trim()) return null;
211
- let parsed;
212
- try {
213
- parsed = xml.parse(text);
214
- } catch {
215
- return null;
216
- }
217
- if (!isRecord(parsed)) return null;
218
- if (isRecord(parsed.rss)) return parseRss(parsed.rss);
219
- if (isRecord(parsed.feed)) return parseAtom(parsed.feed);
220
- const rdf = parsed["rdf:RDF"] ?? parsed.RDF;
221
- if (isRecord(rdf)) return parseRss10(rdf);
222
- return null;
223
- }
224
- function collectItems(value) {
225
- const rawItems = Array.isArray(value) ? value : [];
226
- const items = [];
227
- for (const raw of rawItems) if (isRecord(raw) && readString(raw.title) !== null) items.push({ raw });
228
- return items;
229
- }
230
- function parseRss(rss) {
231
- const { channel } = rss;
232
- if (!isRecord(channel)) return null;
233
- return {
234
- kind: "rss",
235
- title: readString(channel.title),
236
- items: collectItems(channel.item)
237
- };
238
- }
239
- function parseRss10(rdf) {
240
- const channel = isRecord(rdf.channel) ? rdf.channel : null;
241
- return {
242
- kind: "rss",
243
- title: channel ? readString(channel.title) : null,
244
- items: collectItems(rdf.item)
245
- };
246
- }
247
- function parseAtom(feed) {
248
- return {
249
- kind: "atom",
250
- title: readString(feed.title),
251
- items: collectItems(feed.entry)
252
- };
253
- }
254
- function readString(value) {
255
- if (isNonEmptyString(value)) return value;
256
- if (typeof value === "string") return null;
257
- if (isRecord(value)) return readStringFromRecord(value);
258
- if (Array.isArray(value)) return readStringFromArray(value);
259
- return null;
260
- }
261
- function readStringFromRecord(record) {
262
- const text = record["#text"];
263
- if (isNonEmptyString(text)) return text;
264
- const cdata = record["#cdata"];
265
- if (isNonEmptyString(cdata)) return cdata;
266
- return null;
267
- }
268
- function readStringFromArray(array) {
269
- for (const entry of array) {
270
- const resolved = readString(entry);
271
- if (resolved !== null) return resolved;
272
- }
273
- return null;
274
- }
275
- function stripBom(text) {
276
- return text.charCodeAt(0) === 65279 ? text.slice(1) : text;
277
- }
278
- //#endregion
279
- //#region src/feeds/server/pathResolver.ts
280
- var BRACKET_RE = /\[(\d*)\]/g;
281
- /** Split one dot-segment (`results[0]`, `hourly[]`, `name`) into tokens.
282
- * An empty `[]` is an array-identity marker and emits no token — the
283
- * array value is read by the preceding key. */
284
- function parseSegment(segment, tokens) {
285
- const bracketStart = segment.indexOf("[");
286
- const name = bracketStart === -1 ? segment : segment.slice(0, bracketStart);
287
- if (name.length > 0) tokens.push({
288
- kind: "key",
289
- key: name
290
- });
291
- if (bracketStart === -1) return;
292
- for (const match of segment.slice(bracketStart).matchAll(BRACKET_RE)) {
293
- const [, inner] = match;
294
- if (inner.length > 0) tokens.push({
295
- kind: "index",
296
- index: Number(inner)
297
- });
298
- }
299
- }
300
- function tokenize(path) {
301
- const tokens = [];
302
- for (const segment of path.split(".")) if (segment.length > 0) parseSegment(segment, tokens);
303
- return tokens;
304
- }
305
- function step(current, token) {
306
- if (current === null || current === void 0) return void 0;
307
- if (token.kind === "key") {
308
- if (typeof current !== "object" || Array.isArray(current)) return void 0;
309
- return current[token.key];
310
- }
311
- return Array.isArray(current) ? current[token.index] : void 0;
312
- }
313
- /** Resolve a dot/bracket path against `root`, or `undefined` on any miss. */
314
- function getByPath(root, path) {
315
- let current = root;
316
- for (const token of tokenize(path)) current = step(current, token);
317
- return current;
318
- }
319
- /** Locate the array of raw items for a fetch. With `itemsAt` set, walk
320
- * to it; without it, the response itself must be the array. Non-arrays
321
- * yield `[]` so a malformed response is a no-op, not a crash. */
322
- function getItemsArray(root, itemsAt) {
323
- if (!itemsAt) return Array.isArray(root) ? root : [];
324
- const value = getByPath(root, itemsAt);
325
- return Array.isArray(value) ? value : [];
326
- }
327
- //#endregion
328
- //#region src/feeds/server/projectItem.ts
329
- function asKeyString(value) {
330
- if (typeof value === "string" && value.trim().length > 0) return value.trim();
331
- if (typeof value === "number" && Number.isFinite(value)) return String(value);
332
- return null;
333
- }
334
- /** Collapse an XML element object that carries body text alongside
335
- * attributes (`{ "#text": "v", "@_x": "..." }`) or CDATA to its text.
336
- * Generic — the host knows no field names, only this fast-xml-parser
337
- * shape — so a tag like `<guid isPermaLink="false">id</guid>` maps to
338
- * its text without the caller needing to write `.#text`. */
339
- function unwrapTextNode(value) {
340
- if (value && typeof value === "object" && !Array.isArray(value)) {
341
- const obj = value;
342
- if (typeof obj["#text"] === "string") return obj["#text"];
343
- if (typeof obj["#cdata"] === "string") return obj["#cdata"];
344
- }
345
- return value;
346
- }
347
- /** Normalize a mapped value generically — driven by the SCHEMA's declared
348
- * field type, never by the source field name. Today: unwrap XML text
349
- * nodes, and coerce anything mapped into a `date` field to a `YYYY-MM-DD`
350
- * civil date (the collection `date` type + calendar are day-granularity
351
- * and parse strictly — a full RFC-3339 timestamp would be rejected). */
352
- function normalizeValue(value, fieldType) {
353
- const unwrapped = unwrapTextNode(value);
354
- if (fieldType === "date" && typeof unwrapped === "string") {
355
- const millis = Date.parse(unwrapped);
356
- if (Number.isFinite(millis)) return new Date(millis).toISOString().slice(0, 10);
357
- }
358
- return unwrapped;
359
- }
360
- /** Slugify a natural key into a stable, filename-safe id. Short, safe
361
- * keys pass through (lowercased); long or unsafe keys collapse to a
362
- * hash so the filename stays bounded and valid. */
363
- function toSafeId(natural) {
364
- const slug = natural.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
365
- if (slug.length > 0 && slug.length <= 80) return slug;
366
- const hash = (0, node_crypto.createHash)("sha256").update(natural || "item", "utf-8").digest("hex").slice(0, 16);
367
- return slug.length > 80 ? `${slug.slice(0, 60)}-${hash}` : `feed-${hash}`;
368
- }
369
- function naturalKey(record, rawItem, ingest, schema) {
370
- const fromMapped = asKeyString(record[schema.primaryKey]);
371
- if (fromMapped) return fromMapped;
372
- if (ingest.idFrom) {
373
- const fromId = asKeyString(unwrapTextNode(getByPath(rawItem, ingest.idFrom)));
374
- if (fromId) return fromId;
375
- }
376
- return JSON.stringify(record);
377
- }
378
- /** Build a record from a raw fetched item (a parsed RSS/Atom element or a
379
- * JSON object) using `ingest.map`. Each source path is resolved against
380
- * the raw item and normalized per the target field's declared type. The
381
- * returned record's primaryKey is set to the derived safe id (so it
382
- * doubles as the filename). */
383
- function projectRecord(rawItem, ingest, schema) {
384
- const record = {};
385
- for (const [targetField, sourcePath] of Object.entries(ingest.map)) {
386
- const value = normalizeValue(getByPath(rawItem, sourcePath), schema.fields?.[targetField]?.type);
387
- if (value !== void 0) record[targetField] = value;
388
- }
389
- record[schema.primaryKey] = toSafeId(naturalKey(record, rawItem, ingest, schema));
390
- return record;
391
- }
392
- //#endregion
393
- //#region src/feeds/server/retrievers/rss.ts
394
- var retrieveRss = async (ingest, schema) => {
395
- const feed = parseFeed(await fetchText(ingest.url));
396
- if (!feed) return {
397
- items: [],
398
- cursor: {}
399
- };
400
- return {
401
- items: feed.items.map((item) => projectRecord(item.raw, ingest, schema)),
402
- cursor: {}
403
- };
404
- };
405
- registerRetriever("rss", retrieveRss);
406
- registerRetriever("atom", retrieveRss);
407
- //#endregion
408
- //#region src/feeds/server/retrievers/httpJson.ts
409
- var retrieveHttpJson = async (ingest, schema) => {
410
- return {
411
- items: getItemsArray(await fetchJson(ingest.url), ingest.itemsAt).map((raw) => projectRecord(raw, ingest, schema)),
412
- cursor: {}
413
- };
414
- };
415
- registerRetriever("http-json", retrieveHttpJson);
416
- //#endregion
417
- //#region src/feeds/server/state.ts
418
- /** Resolve the on-disk state file for a collection, branching on source. */
419
- function stateFilePath(target, workspaceRoot) {
420
- return target.source === "feed" ? require_feeds_paths.feedStatePath(target.slug, workspaceRoot) : require_feeds_paths.ingestStatePath(target.slug, workspaceRoot);
421
- }
422
- function defaultFeedState(slug) {
423
- return {
424
- slug,
425
- lastFetchedAt: null,
426
- cursor: {},
427
- consecutiveFailures: 0
428
- };
429
- }
430
- function normalizeState(slug, parsed) {
431
- const base = defaultFeedState(slug);
432
- const cursor = parsed.cursor && typeof parsed.cursor === "object" ? parsed.cursor : base.cursor;
433
- return {
434
- slug,
435
- lastFetchedAt: typeof parsed.lastFetchedAt === "string" ? parsed.lastFetchedAt : base.lastFetchedAt,
436
- cursor,
437
- consecutiveFailures: typeof parsed.consecutiveFailures === "number" ? parsed.consecutiveFailures : base.consecutiveFailures,
438
- ...typeof parsed.failureBellId === "string" ? { failureBellId: parsed.failureBellId } : {}
439
- };
440
- }
441
- /** Read a collection's retrieval state, tolerating a missing file (first run →
442
- * default). The state path branches on `target.source`. */
443
- async function readFeedState(workspaceRoot, target) {
444
- const { slug } = target;
445
- try {
446
- const raw = await (0, node_fs_promises.readFile)(stateFilePath(target, workspaceRoot), "utf-8");
447
- return normalizeState(slug, JSON.parse(raw));
448
- } catch (err) {
449
- if (err.code !== "ENOENT") log.warn("feeds", "failed to read feed state, using default", {
450
- slug,
451
- error: String(err)
452
- });
453
- return defaultFeedState(slug);
454
- }
455
- }
456
- /** Persist a collection's retrieval state atomically (creating the parent dir
457
- * if needed). The state path branches on `target.source`. */
458
- async function writeFeedState(workspaceRoot, target, state) {
459
- const file = stateFilePath(target, workspaceRoot);
460
- await (0, node_fs_promises.mkdir)(node_path.default.dirname(file), { recursive: true });
461
- await requireFeedsHost().writeFileAtomic(file, `${JSON.stringify(state, null, 2)}\n`);
462
- }
463
- //#endregion
464
- //#region src/feeds/server/agentIngest.ts
465
- /** The injected worker launcher, or null if the host was never configured.
466
- * Read non-throwingly so the failure-isolated contract holds (an unconfigured
467
- * host becomes an `errors` entry, not a thrown exception). */
468
- function workerRunnerOrNull() {
469
- try {
470
- return requireFeedsHost().spawnWorker;
471
- } catch {
472
- return null;
473
- }
474
- }
475
- function result(slug, patch) {
476
- return {
477
- slug,
478
- written: 0,
479
- removed: 0,
480
- errors: [],
481
- ...patch
482
- };
483
- }
484
- /** Dispatch one agent-ingest refresh: build the seed, launch a worker, and (on a
485
- * successful launch) stamp `lastFetchedAt` with the DISPATCH time — not
486
- * completion. That's what gates the due-loop, so a slow worker can't cause a
487
- * double-dispatch. `opts.hidden` (default true) runs an invisible system worker
488
- * for SCHEDULED refreshes; a MANUAL Refresh passes `hidden:false` for a visible,
489
- * debuggable session. Failure-isolated: never throws; cap-miss / template-miss /
490
- * launch error leave state untouched and report via `errors`. */
491
- async function refreshViaAgent(workspaceRoot, collection, opts) {
492
- const hidden = opts?.hidden ?? true;
493
- const { slug } = collection;
494
- const ingest = collection.schema.ingest;
495
- if (!ingest || ingest.kind !== "agent") return result(slug, { errors: ["collection has no agent ingest config"] });
496
- const workerRunner = workerRunnerOrNull();
497
- if (!workerRunner) return result(slug, { errors: ["agent ingest worker runner not configured"] });
498
- const template = await require_server.readSkillTemplate(collection.skillDir, ingest.template);
499
- if (template === null) return result(slug, { errors: [`ingest template '${ingest.template}' could not be read`] });
500
- const items = await require_server.listItems(collection.dataDir, { workspaceRoot });
501
- const message = require_server.buildCollectionActionSeedPrompt(items, collection.schema, template);
502
- let launch;
503
- try {
504
- launch = await workerRunner({
505
- message,
506
- roleId: ingest.role,
507
- hidden,
508
- onComplete: hidden ? (outcome) => recordOutcome(workspaceRoot, collection, outcome.didError) : void 0
509
- });
510
- } catch (err) {
511
- log.warn("feeds", "agent ingest worker launch threw", {
512
- slug,
513
- error: String(err)
514
- });
515
- return result(slug, {
516
- errors: [String(err)],
517
- dispatched: false
518
- });
519
- }
520
- if (!launch.ok) {
521
- log.info("feeds", "agent ingest dispatch skipped", {
522
- slug,
523
- error: launch.error
524
- });
525
- return result(slug, {
526
- errors: [launch.error],
527
- dispatched: false
528
- });
529
- }
530
- await writeFeedState(workspaceRoot, collection, {
531
- ...await readFeedState(workspaceRoot, collection),
532
- lastFetchedAt: (/* @__PURE__ */ new Date()).toISOString()
533
- });
534
- log.info("feeds", "agent ingest dispatched", {
535
- slug,
536
- role: ingest.role,
537
- chatId: launch.chatId,
538
- hidden,
539
- items: items.length
540
- });
541
- return result(slug, {
542
- dispatched: true,
543
- ...hidden ? {} : { chatId: launch.chatId }
544
- });
545
- }
546
- /** Completion hook: reconcile failure tracking when a dispatched worker
547
- * finishes. On error, bump `consecutiveFailures` and raise a single failure
548
- * bell (deduped via the persisted `failureBellId`). On success, reset the
549
- * counter and clear any standing bell. Best-effort + failure-isolated — it
550
- * runs inside the agent run's teardown, so it must never throw. */
551
- async function recordOutcome(workspaceRoot, collection, didError) {
552
- const state = await readFeedState(workspaceRoot, collection);
553
- await writeFeedState(workspaceRoot, collection, didError ? await onWorkerError(state, collection) : await onWorkerSuccess(state, collection));
554
- }
555
- async function onWorkerError(state, collection) {
556
- const next = {
557
- ...state,
558
- consecutiveFailures: state.consecutiveFailures + 1
559
- };
560
- log.warn("feeds", "agent ingest worker failed", {
561
- slug: collection.slug,
562
- consecutiveFailures: next.consecutiveFailures
563
- });
564
- if (!state.failureBellId) try {
565
- const { id } = await require_notifier.publish({
566
- pluginPkg: "host",
567
- severity: "nudge",
568
- lifecycle: "fyi",
569
- title: "Collection refresh failed",
570
- body: `“${collection.schema.title}” (${collection.slug}) couldn't refresh. Open it to retry.`,
571
- navigateTarget: `/collections/${collection.slug}`
572
- });
573
- next.failureBellId = id;
574
- } catch (err) {
575
- log.warn("feeds", "failed to publish ingest failure bell", {
576
- slug: collection.slug,
577
- error: String(err)
578
- });
579
- }
580
- return next;
581
- }
582
- async function onWorkerSuccess(state, collection) {
583
- log.info("feeds", "agent ingest worker completed", { slug: collection.slug });
584
- if (state.failureBellId) await require_notifier.clear(state.failureBellId).catch((err) => log.warn("feeds", "failed to clear ingest failure bell", {
585
- slug: collection.slug,
586
- error: String(err)
587
- }));
588
- const next = {
589
- ...state,
590
- consecutiveFailures: 0
591
- };
592
- delete next.failureBellId;
593
- return next;
594
- }
595
- //#endregion
596
- //#region src/feeds/server/engine.ts
597
- var ONE_HOUR_MS = 36e5;
598
- var ONE_DAY_MS = 864e5;
599
- /** Feed schemas carry the rich `IngestSpec` (validated at discovery —
600
- * `source === "feed"` requires `ingest`), but the canonical
601
- * `CollectionSchema.ingest` only promises the minimal `CollectionIngest`.
602
- * Narrow here so the engine can read the retrieval fields type-safely. */
603
- function feedIngest(schema) {
604
- return schema.ingest;
605
- }
606
- async function upsertItems(workspaceRoot, feed, items) {
607
- let written = 0;
608
- for (const item of items) {
609
- const itemId = item[feed.schema.primaryKey];
610
- if (typeof itemId !== "string" || itemId.length === 0) continue;
611
- const result = await require_server.writeItem(feed.dataDir, itemId, item, {
612
- refuseOverwrite: false,
613
- workspaceRoot,
614
- slug: feed.slug
615
- });
616
- if (result.kind === "ok") written += 1;
617
- else log.warn("feeds", "feed item write skipped", {
618
- slug: feed.slug,
619
- itemId,
620
- kind: result.kind
621
- });
622
- }
623
- return written;
624
- }
625
- /** The schema's first `date` field, used to order records for the
626
- * maxItems cap. Null when the schema declares none. */
627
- function firstDateField(schema) {
628
- for (const [key, spec] of Object.entries(schema.fields)) if (spec.type === "date") return key;
629
- return null;
630
- }
631
- function recordTime(item, field) {
632
- const value = item[field];
633
- const millis = typeof value === "string" ? Date.parse(value) : NaN;
634
- return Number.isFinite(millis) ? millis : Number.NEGATIVE_INFINITY;
635
- }
636
- /** Enforce `ingest.maxItems` (default 100): keep the newest N records by
637
- * the schema's date field, delete the rest. No-op when the cap is 0/absent
638
- * of a date field, or when under the cap. Returns the number deleted. */
639
- async function pruneFeed(workspaceRoot, feed) {
640
- const ingest = feedIngest(feed.schema);
641
- const cap = (ingest && ingest.kind !== "agent" ? ingest.maxItems : void 0) ?? 100;
642
- if (cap <= 0) return 0;
643
- const dateField = firstDateField(feed.schema);
644
- if (!dateField) {
645
- log.warn("feeds", "maxItems prune skipped: schema has no date field to order by", { slug: feed.slug });
646
- return 0;
647
- }
648
- const items = await require_server.listItems(feed.dataDir, { workspaceRoot });
649
- if (items.length <= cap) return 0;
650
- const stale = [...items].sort((left, right) => recordTime(right, dateField) - recordTime(left, dateField)).slice(cap);
651
- let removed = 0;
652
- for (const item of stale) {
653
- const itemId = item[feed.schema.primaryKey];
654
- if (typeof itemId !== "string" || itemId.length === 0) continue;
655
- if ((await require_server.deleteItem(feed.dataDir, itemId, {
656
- workspaceRoot,
657
- slug: feed.slug
658
- })).kind === "ok") removed += 1;
659
- }
660
- if (removed > 0) log.info("feeds", "pruned old feed records", {
661
- slug: feed.slug,
662
- removed,
663
- cap
664
- });
665
- return removed;
666
- }
667
- /** Fetch one feed now, upsert its records, then enforce the maxItems cap.
668
- * Failure-isolated: returns an errors array rather than throwing. */
669
- async function refreshOne(workspaceRoot, feed, opts) {
670
- const { slug } = feed;
671
- const ingest = feedIngest(feed.schema);
672
- if (!ingest) return {
673
- slug,
674
- written: 0,
675
- removed: 0,
676
- errors: ["collection has no ingest config"]
677
- };
678
- if (ingest.kind === "agent") return refreshViaAgent(workspaceRoot, feed, opts);
679
- const retriever = getRetriever(ingest.kind);
680
- if (!retriever) return {
681
- slug,
682
- written: 0,
683
- removed: 0,
684
- errors: [`no retriever registered for kind '${ingest.kind}'`]
685
- };
686
- const state = await readFeedState(workspaceRoot, feed);
687
- try {
688
- const result = await retriever(ingest, feed.schema, state);
689
- const written = await upsertItems(workspaceRoot, feed, result.items);
690
- await writeFeedState(workspaceRoot, feed, {
691
- ...state,
692
- lastFetchedAt: (/* @__PURE__ */ new Date()).toISOString(),
693
- cursor: result.cursor,
694
- consecutiveFailures: 0
695
- });
696
- const removed = await pruneFeed(workspaceRoot, feed);
697
- log.info("feeds", "feed refreshed", {
698
- slug,
699
- written,
700
- removed,
701
- fetched: result.items.length
702
- });
703
- return {
704
- slug,
705
- written,
706
- removed,
707
- errors: []
708
- };
709
- } catch (error) {
710
- await writeFeedState(workspaceRoot, feed, {
711
- ...state,
712
- consecutiveFailures: state.consecutiveFailures + 1
713
- });
714
- const message = String(error);
715
- log.warn("feeds", "feed refresh failed", {
716
- slug,
717
- error: message
718
- });
719
- return {
720
- slug,
721
- written: 0,
722
- removed: 0,
723
- errors: [message]
724
- };
725
- }
726
- }
727
- function dueIntervalMs(schedule) {
728
- switch (schedule) {
729
- case "daily": return ONE_DAY_MS;
730
- case "weekly": return 7 * ONE_DAY_MS;
731
- default: return ONE_HOUR_MS;
732
- }
733
- }
734
- /** True iff a `daily` schedule with a UTC `atHour` anchor is due. The system
735
- * task ticks hourly, so "due" means: we're in the anchor hour AND we haven't
736
- * already run in roughly the last day. The 23 h floor (not 24 h) tolerates the
737
- * tick landing a few minutes earlier than the previous run, while staying well
738
- * above 1 h so a second tick in the same anchor hour can't double-fire. */
739
- function isDailyAtHourDue(now, atHour, lastFetchedAt) {
740
- if (now.getUTCHours() !== atHour) return false;
741
- if (!lastFetchedAt) return true;
742
- const elapsed = now.getTime() - Date.parse(lastFetchedAt);
743
- if (!Number.isFinite(elapsed)) return true;
744
- return elapsed >= 23 * ONE_HOUR_MS;
745
- }
746
- /** True iff a collection is due to refresh given its schedule + last run.
747
- * `on-demand` is never auto-due; a `daily` schedule with `atHour` anchors to
748
- * that UTC hour, otherwise cadence is elapsed-based. */
749
- function isFeedDue(feed, state) {
750
- const ingest = feedIngest(feed.schema);
751
- const schedule = ingest?.schedule;
752
- if (!schedule || schedule === "on-demand") return false;
753
- if (schedule === "daily" && typeof ingest?.atHour === "number") return isDailyAtHourDue(/* @__PURE__ */ new Date(), ingest.atHour, state.lastFetchedAt);
754
- if (!state.lastFetchedAt) return true;
755
- const elapsed = Date.now() - Date.parse(state.lastFetchedAt);
756
- if (!Number.isFinite(elapsed)) return true;
757
- return elapsed >= dueIntervalMs(schedule);
758
- }
759
- /** Refresh every collection whose ingest schedule says it's due — declarative
760
- * feeds AND skill-backed collections with `ingest.kind: "agent"`. Called by the
761
- * hourly system task. Sequential + failure-isolated. */
762
- async function refreshDue(workspaceRoot = requireFeedsHost().workspaceRoot) {
763
- const withIngest = (await require_server.discoverCollections({ workspaceRoot })).filter((collection) => collection.schema.ingest);
764
- const results = [];
765
- for (const collection of withIngest) try {
766
- if (!isFeedDue(collection, await readFeedState(workspaceRoot, collection))) continue;
767
- results.push(await refreshOne(workspaceRoot, collection));
768
- } catch (error) {
769
- log.warn("feeds", "scheduled refresh failed for collection", {
770
- slug: collection.slug,
771
- error: String(error)
772
- });
773
- results.push({
774
- slug: collection.slug,
775
- written: 0,
776
- removed: 0,
777
- errors: [String(error)]
778
- });
779
- }
780
- return results;
781
- }
782
- //#endregion
783
- exports.AGENT_INGEST_KIND = require_ingestTypes.AGENT_INGEST_KIND;
784
- exports.DEFAULT_FEED_MAX_ITEMS = require_ingestTypes.DEFAULT_FEED_MAX_ITEMS;
785
- exports.FEEDS_DIR = require_feeds_paths.FEEDS_DIR;
786
- exports.FEED_SCHEDULES = require_deriveAll.FEED_SCHEDULES;
787
- exports.INGEST_KINDS = require_deriveAll.INGEST_KINDS;
788
- exports.configureFeedsHost = configureFeedsHost;
789
- exports.feedDir = require_feeds_paths.feedDir;
790
- exports.feedStatePath = require_feeds_paths.feedStatePath;
791
- exports.feedsRoot = require_feeds_paths.feedsRoot;
792
- exports.ingestStateDir = require_feeds_paths.ingestStateDir;
793
- exports.ingestStatePath = require_feeds_paths.ingestStatePath;
794
- exports.isFeedSchedule = require_ingestTypes.isFeedSchedule;
795
- exports.listFeeds = listFeeds;
796
- exports.readFeedState = readFeedState;
797
- exports.refreshDue = refreshDue;
798
- exports.refreshOne = refreshOne;
799
- exports.refreshViaAgent = refreshViaAgent;
800
- exports.removeFeed = removeFeed;
801
- exports.requireFeedsHost = requireFeedsHost;
802
- exports.resetFeedsHostForTesting = resetFeedsHostForTesting;
803
-
804
- //# sourceMappingURL=index.cjs.map