@joe-broadhead/opencode-mem 0.9.12 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1934 -2
- package/README.md +29 -17
- package/SECURITY.md +3 -1
- package/docs/agent-instructions.md +17 -0
- package/docs/agent-protocol.md +8 -8
- package/docs/permissions.md +22 -4
- package/package.json +7923 -217
- package/shared/page-paths.js +44 -0
- package/skills/mem.pages/SKILL.md +8 -1
- package/skills/mem.usage/SKILL.md +15 -15
- package/src/action-grant-keys.js +159 -0
- package/src/action-grant.js +439 -0
- package/src/actions.js +1690 -0
- package/src/backend/records.js +25 -2
- package/src/backend/v3.js +76 -22
- package/src/call-shapes.js +17 -16
- package/src/collector-lock.js +533 -62
- package/src/collector-service.js +153 -26
- package/src/collector.js +627 -33
- package/src/config.js +79 -6
- package/src/connect.js +106 -8
- package/src/credentials.js +123 -25
- package/src/events.js +174 -3
- package/src/forget-turn.js +141 -0
- package/src/host-refusal.js +78 -0
- package/src/index.js +209 -22
- package/src/jobs/host.js +557 -0
- package/src/ops/codes.js +110 -0
- package/src/ops/copy-lineage.js +42 -0
- package/src/ops/index.js +28 -0
- package/src/ops/space-ops.js +1936 -0
- package/src/outlinks.js +103 -0
- package/src/p2-reservation.js +151 -0
- package/src/permissions.js +30 -0
- package/src/plugins/descriptor.js +62 -0
- package/src/plugins/discovery.js +318 -0
- package/src/plugins/grants.js +178 -0
- package/src/plugins/locate.js +319 -0
- package/src/plugins/manifest.js +1162 -0
- package/src/plugins/source-contract/index.js +34 -0
- package/src/plugins/source-contract/policy.js +102 -0
- package/src/plugins/source-contract/vendor/PROVENANCE.json +17 -0
- package/src/plugins/source-contract/vendor/RELEASE-SHA256SUMS +3 -0
- package/src/plugins/source-contract/vendor/RELEASE.json +17 -0
- package/src/plugins/source-contract/vendor/SHA256SUMS +41 -0
- package/src/plugins/source-contract/vendor/THIRD-PARTY-NOTICES.md +35 -0
- package/src/plugins/source-contract/vendor/broker-client.js +93 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-3M5PAKSS.js +84 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-4ERCA4TK.js +185 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-6HDIB2SA.js +183 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-ELPJD3ND.js +10 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-HLVAKAYG.js +10 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-I2RXJJJM.js +81 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-KWEWSZ4V.js +505 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-KYWBTAF4.js +182 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-Q3RQA5J2.js +249 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-RRUAIQ4I.js +156 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-RYNKNCPP.js +165 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-W5SHR7DQ.js +9477 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-ZUKF7P6U.js +653 -0
- package/src/plugins/source-contract/vendor/cursor.js +26 -0
- package/src/plugins/source-contract/vendor/descriptor.js +34 -0
- package/src/plugins/source-contract/vendor/events.js +257 -0
- package/src/plugins/source-contract/vendor/index.js +10 -0
- package/src/plugins/source-contract/vendor/params.js +22 -0
- package/src/plugins/source-contract/vendor/permission.js +28 -0
- package/src/plugins/source-contract/vendor/pick.js +22 -0
- package/src/plugins/source-contract/vendor/read.js +29 -0
- package/src/plugins/source-contract/vendor/registry.js +23 -0
- package/src/plugins/source-contract/vendor/schema.js +17 -0
- package/src/plugins/source-contract/vendor/schemas/output.agenda.json +96 -0
- package/src/plugins/source-contract/vendor/schemas/output.countdown.json +28 -0
- package/src/plugins/source-contract/vendor/schemas/output.image.json +28 -0
- package/src/plugins/source-contract/vendor/schemas/output.inbox.json +84 -0
- package/src/plugins/source-contract/vendor/schemas/output.links.json +46 -0
- package/src/plugins/source-contract/vendor/schemas/output.list.json +57 -0
- package/src/plugins/source-contract/vendor/schemas/output.metric.json +61 -0
- package/src/plugins/source-contract/vendor/schemas/output.note.json +14 -0
- package/src/plugins/source-contract/vendor/schemas/output.progress.json +50 -0
- package/src/plugins/source-contract/vendor/schemas/output.record.json +38 -0
- package/src/plugins/source-contract/vendor/schemas/output.table.json +61 -0
- package/src/plugins/source-contract/vendor/schemas/output.thread.json +62 -0
- package/src/plugins/source-contract/vendor/schemas/sources.json +819 -0
- package/src/plugins/source-contract/vendor/schemas.js +19 -0
- package/src/plugins/source-contract/vendor/tools-rpc.js +42 -0
- package/src/plugins/verify.js +56 -0
- package/src/protocol.js +21 -5
- package/src/recall.js +104 -5
- package/src/records.js +61 -2
- package/src/requests.js +216 -0
- package/src/runtime.js +89 -7
- package/src/setup-time.js +68 -0
- package/src/skills.js +230 -8
- package/src/source-ids.js +25 -0
- package/src/sources/declaration.js +365 -0
- package/src/sources/reads.js +772 -0
- package/src/sources/rpc.js +151 -0
- package/src/sources.js +409 -90
- package/src/spaces.js +283 -11
- package/src/spool.js +61 -4
- package/src/stated.js +10 -5
- package/src/storable.js +139 -0
- package/src/tools/commitments.js +47 -5
- package/src/tools/held-write.js +36 -13
- package/src/tools/index.js +27 -0
- package/src/tools/ingest.js +27 -2
- package/src/tools/memory.js +52 -14
- package/src/tools/page.js +95 -9
- package/src/tools/recall.js +208 -4
- package/src/tools/records-kit.js +2 -1
- package/src/tools/share.js +42 -3
- package/src/tools/shared.js +44 -4
- package/src/tools/space-lifecycle.js +278 -0
- package/src/tools/spaces.js +373 -29
- package/src/tools/status.js +9 -2
- package/src/tools/wiki.js +34 -4
- package/src/util.js +110 -6
- package/src/vendor/source-contract/README.md +24 -0
- package/src/vendor/source-contract/THIRD-PARTY-NOTICES.md +35 -0
- package/src/vendor/source-contract/VENDOR.json +81 -0
- package/src/vendor/source-contract/chunks/chunk-3M5PAKSS.js +84 -0
- package/src/vendor/source-contract/chunks/chunk-6HDIB2SA.js +183 -0
- package/src/vendor/source-contract/chunks/chunk-HLVAKAYG.js +10 -0
- package/src/vendor/source-contract/chunks/chunk-I2RXJJJM.js +81 -0
- package/src/vendor/source-contract/chunks/chunk-KWEWSZ4V.js +505 -0
- package/src/vendor/source-contract/chunks/chunk-W5SHR7DQ.js +9477 -0
- package/src/vendor/source-contract/chunks/chunk-ZUKF7P6U.js +653 -0
- package/src/vendor/source-contract/cursor.js +26 -0
- package/src/vendor/source-contract/events.js +257 -0
- package/src/vendor/source-contract/registry.js +23 -0
- package/src/vendor/source-contract/tools-rpc.js +42 -0
- package/src/views/access-reads.js +256 -0
- package/src/views/communities.js +77 -0
- package/src/views/contracts.js +1183 -36
- package/src/views/inbox-bytes.js +103 -0
- package/src/views/overdue.js +1 -1
- package/src/views/profile.js +31 -0
- package/src/views/receipt.js +81 -0
- package/src/views/records-reads.js +869 -37
- package/src/views/rpc.js +159 -11
- package/src/views/shape.js +8 -5
- package/views/memory.html +16 -3
- package/views/pages.html +2 -2
- package/views/remembered.html +130 -0
package/src/collector.js
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { errorMessage, isPlainObject, oneLine } from "./util.js"
|
|
2
2
|
import { createHash } from "node:crypto"
|
|
3
|
-
import {
|
|
3
|
+
import { isSourceName } from "./source-ids.js"
|
|
4
|
+
import { ACCOUNT_NAME, DEFAULT_EVERY_MINUTES, contractRefusal, rpcDefinition, sourceAllowRules, sourceEntry } from "./sources.js"
|
|
5
|
+
import { acceptAnswer, probe, read } from "./plugins/source-contract/vendor/read.js"
|
|
6
|
+
import { isBrokerAnswer, sourceReader } from "./plugins/source-contract/vendor/broker-client.js"
|
|
7
|
+
import { probeResultProblems } from "./plugins/source-contract/vendor/events.js"
|
|
8
|
+
import { cursorAdvance, failureAction, failureFromError } from "./plugins/source-contract/policy.js"
|
|
9
|
+
import { memoryResourceAvailable } from "./credentials.js"
|
|
4
10
|
|
|
5
11
|
/**
|
|
6
12
|
* The memory collector (docs/design/memory-v1-contracts.md §4.4, §4.5).
|
|
@@ -29,9 +35,12 @@ import { ACCOUNT_NAME, DEFAULT_EVERY_MINUTES, SOURCE_REGISTRY, contractRefusal,
|
|
|
29
35
|
* that adds the API does not reach it either. `collectCheckCode` and
|
|
30
36
|
* `createCollector({scheduler})` stay as pure units; nothing wires a scheduler.
|
|
31
37
|
*
|
|
32
|
-
* Every `events_list` answer must carry `
|
|
33
|
-
*
|
|
34
|
-
*
|
|
38
|
+
* Every `events_list` answer must carry `account` (§4.5); one without it is
|
|
39
|
+
* refused with the release its plugin needs. The ingest is keyed by the
|
|
40
|
+
* binding's own source and the answer's account: a declared source's page names
|
|
41
|
+
* the plugin's name for it (`slack`), while the person confirmed the
|
|
42
|
+
* descriptor's id (`slack.workspace`), which is what the store holds the
|
|
43
|
+
* consent and the cursor under — the same key a job uses.
|
|
35
44
|
*/
|
|
36
45
|
|
|
37
46
|
export const COLLECT_SCHEDULE = "memory-collect"
|
|
@@ -66,9 +75,10 @@ const freezeRules = (rules) => Object.freeze(rules.map((rule) => Object.freeze({
|
|
|
66
75
|
* `collectSessionRules({source, account})`: one binding's session, which the
|
|
67
76
|
* in-plugin collector and a bind's probe both use. It allows the source's
|
|
68
77
|
* events action, the exact inner reads its events_list asserts (OpenCode
|
|
69
|
-
* checks each against this session: S0, 0.9.11),
|
|
70
|
-
*
|
|
71
|
-
*
|
|
78
|
+
* checks each against this session: S0, 0.9.11), for gws the Gmail operations
|
|
79
|
+
* of its EVENT_READS on the binding's own account, and for github the four
|
|
80
|
+
* `github_api` GETs its events_list makes. There is no session for every
|
|
81
|
+
* source at once (0.9.7).
|
|
72
82
|
*/
|
|
73
83
|
export function collectSessionRules({ source, account }) {
|
|
74
84
|
const entry = sourceEntry(source)
|
|
@@ -107,13 +117,11 @@ export function sessionEffect(action, resource = "*", rules = []) {
|
|
|
107
117
|
return rules.findLast((rule) => wildcard(rule.action, action) && wildcard(rule.resource, resource))?.effect ?? "ask"
|
|
108
118
|
}
|
|
109
119
|
|
|
110
|
-
/**
|
|
111
|
-
const SOURCE_NAME = /^[a-z][a-z0-9_-]{0,63}$/
|
|
120
|
+
/** An identifier in a declaration or a template. */
|
|
112
121
|
const IDENTIFIER = /^[a-zA-Z][a-zA-Z0-9_]*$/
|
|
113
122
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
}
|
|
123
|
+
// A binding's source: a built-in name, or a declared source's contract id (src/source-ids.js).
|
|
124
|
+
export { isSourceName } from "./source-ids.js"
|
|
117
125
|
|
|
118
126
|
export function isAccountName(value) {
|
|
119
127
|
return typeof value === "string" && ACCOUNT_NAME.test(value)
|
|
@@ -140,7 +148,9 @@ export function collectableSources(sources, { only } = {}) {
|
|
|
140
148
|
if (entry.status !== "confirmed") continue
|
|
141
149
|
const source = String(entry.source ?? "")
|
|
142
150
|
const account = String(entry.account ?? "") || DEFAULT_ACCOUNT
|
|
143
|
-
|
|
151
|
+
const known = sourceEntry(source)
|
|
152
|
+
// A snapshot source is read on demand (`snapshot_read`), never collected as events.
|
|
153
|
+
if (!known || known.kind !== "events" || !isAccountName(account)) continue
|
|
144
154
|
if (only && source !== only) continue
|
|
145
155
|
const binding = { source, account }
|
|
146
156
|
seen.set(bindingKey(binding), binding)
|
|
@@ -159,13 +169,23 @@ function accountInput(entry, account) {
|
|
|
159
169
|
return named !== DEFAULT_ACCOUNT || entry?.namesAccount ? { account: named } : {}
|
|
160
170
|
}
|
|
161
171
|
|
|
162
|
-
|
|
172
|
+
/**
|
|
173
|
+
* One page's input: the cursor, the account, the source's own input, and **the
|
|
174
|
+
* scope the person confirmed**. A scoped source that is not told its scope reads
|
|
175
|
+
* its plugin's whole option, and its page is then refused as out of scope.
|
|
176
|
+
*
|
|
177
|
+
* @param {{ source: string, account?: string }} binding
|
|
178
|
+
* @param {string | undefined} cursor
|
|
179
|
+
* @param {Record<string, string[]>} [scope] the consented scope (`events.cursor` returns it)
|
|
180
|
+
*/
|
|
181
|
+
export function listInput(binding, cursor, scope) {
|
|
163
182
|
const entry = sourceEntry(binding.source)
|
|
164
183
|
return {
|
|
165
184
|
...(cursor ? { cursor } : {}),
|
|
166
185
|
limit: PAGE_LIMIT,
|
|
167
186
|
...accountInput(entry, binding.account),
|
|
168
187
|
...(entry?.input ?? {}),
|
|
188
|
+
...(isPlainObject(scope) && Object.keys(scope).length ? { scope } : {}),
|
|
169
189
|
}
|
|
170
190
|
}
|
|
171
191
|
|
|
@@ -243,7 +263,8 @@ export class SourceContractError extends Error {
|
|
|
243
263
|
* entry's) and `account`. Throws a SourceContractError otherwise; null when it
|
|
244
264
|
* is not an answer at all.
|
|
245
265
|
*/
|
|
246
|
-
export function readEventsAnswer(value, entry) {
|
|
266
|
+
export function readEventsAnswer(value, entry, boundAccount = null) {
|
|
267
|
+
if (entry?.contract === "sources") return readDeclaredAnswer(value, entry, boundAccount)
|
|
247
268
|
let answer = value
|
|
248
269
|
// A wrapper's text beside its output: a source's own words, such as Slack's "No Slack channel is enabled".
|
|
249
270
|
const said = isPlainObject(answer) && isPlainObject(answer.output) && typeof (answer.content ?? answer.text) === "string" ? String(answer.content ?? answer.text) : ""
|
|
@@ -273,6 +294,59 @@ export function readEventsAnswer(value, entry) {
|
|
|
273
294
|
}
|
|
274
295
|
}
|
|
275
296
|
|
|
297
|
+
/**
|
|
298
|
+
* A declared source's answer, checked by the contract's own `acceptAnswer`: the
|
|
299
|
+
* page's source and account, `trust: "untrusted"`, every event's shape, the
|
|
300
|
+
* size against its `maxBytes`, and `refused`/`failed`/`gap`/`cost`. A failure
|
|
301
|
+
* carries the contract's code, so the collector can act on it (§8.3.4).
|
|
302
|
+
*/
|
|
303
|
+
function readDeclaredAnswer(value, entry, account) {
|
|
304
|
+
const outcome = acceptAnswer(entry.item, account || DEFAULT_ACCOUNT, unwrapResult(value))
|
|
305
|
+
if (!outcome.ok) {
|
|
306
|
+
const error = new Error(`${outcome.failure.code}: ${outcome.failure.message}`)
|
|
307
|
+
error.code = outcome.failure.code
|
|
308
|
+
error.next = outcome.failure.next
|
|
309
|
+
throw error
|
|
310
|
+
}
|
|
311
|
+
const page = outcome.page
|
|
312
|
+
return {
|
|
313
|
+
// The binding's id (`gws.gmail`), never the plugin's short page name (`gmail`).
|
|
314
|
+
source: entry.source,
|
|
315
|
+
pageSource: page.source,
|
|
316
|
+
account: page.account,
|
|
317
|
+
owner: page.owner ?? null,
|
|
318
|
+
events: page.events,
|
|
319
|
+
cursor: page.cursor ?? "",
|
|
320
|
+
more: page.more === true,
|
|
321
|
+
refused: Array.isArray(page.refused) ? page.refused : [],
|
|
322
|
+
failed: Array.isArray(page.failed) ? page.failed : [],
|
|
323
|
+
// The page's own echo of what it read: kept, so the store can refuse a page
|
|
324
|
+
// that read wider than the person confirmed (plugin_scope).
|
|
325
|
+
scope: isPlainObject(page.scope) ? page.scope : null,
|
|
326
|
+
gap: isPlainObject(page.gap) ? page.gap : null,
|
|
327
|
+
cost: isPlainObject(page.cost) ? page.cost : null,
|
|
328
|
+
setup: null,
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** The payload of an RPC result: its `output`, its structured content, or the JSON in its text. */
|
|
333
|
+
function unwrapResult(value) {
|
|
334
|
+
let answer = value
|
|
335
|
+
if (isPlainObject(answer) && !Array.isArray(answer.events)) {
|
|
336
|
+
if (isPlainObject(answer.output)) answer = answer.output
|
|
337
|
+
else if (isPlainObject(answer.structuredContent)) answer = answer.structuredContent
|
|
338
|
+
else if (typeof (answer.content ?? answer.text) === "string") answer = answer.content ?? answer.text
|
|
339
|
+
}
|
|
340
|
+
if (typeof answer === "string") {
|
|
341
|
+
try {
|
|
342
|
+
answer = JSON.parse(answer)
|
|
343
|
+
} catch {
|
|
344
|
+
return value
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
return answer
|
|
348
|
+
}
|
|
349
|
+
|
|
276
350
|
const SETUP_WORDS = /\b(?:is|are) (?:not )?(?:enabled|configured|set)\b|\bset the plugin option\b|\bsign in\b|\bnot configured\b/i
|
|
277
351
|
|
|
278
352
|
/**
|
|
@@ -283,11 +357,20 @@ const SETUP_WORDS = /\b(?:is|are) (?:not )?(?:enabled|configured|set)\b|\bset th
|
|
|
283
357
|
*/
|
|
284
358
|
export function setupNote(answer, said = "") {
|
|
285
359
|
const own = typeof answer?.setup === "string" ? answer.setup.trim() : ""
|
|
286
|
-
if (own) return
|
|
287
|
-
if (said && Array.isArray(answer?.events) && answer.events.length === 0 && SETUP_WORDS.test(said)) return
|
|
360
|
+
if (own) return redactedSetup(own)
|
|
361
|
+
if (said && Array.isArray(answer?.events) && answer.events.length === 0 && SETUP_WORDS.test(said)) return redactedSetup(said.split(/(?<=\.)\s/)[0])
|
|
288
362
|
return null
|
|
289
363
|
}
|
|
290
364
|
|
|
365
|
+
/**
|
|
366
|
+
* A source's own setup sentence, redacted: it is third-party text, and it is
|
|
367
|
+
* kept in the run's result, in the status file and in every answer that shows
|
|
368
|
+
* the run (0.10.4's gate; review F2).
|
|
369
|
+
*/
|
|
370
|
+
function redactedSetup(text) {
|
|
371
|
+
return oneLine(errorMessage(text), 400)
|
|
372
|
+
}
|
|
373
|
+
|
|
291
374
|
/** The input of the owner-only call a bind request makes (§4.5): `limit: 0` where the plugin keeps it, else `limit: 1`. */
|
|
292
375
|
export function probeInput(entry, account) {
|
|
293
376
|
return { limit: entry.ownerProbe === "limit0" ? 0 : 1, ...accountInput(entry, account) }
|
|
@@ -327,24 +410,385 @@ export function requestedSources(requests, { only, now = Date.now() } = {}) {
|
|
|
327
410
|
*
|
|
328
411
|
* @returns {null | ((source: string, input: object, sessionID?: string) => Promise<ReturnType<typeof readEventsAnswer>>)}
|
|
329
412
|
*/
|
|
330
|
-
export function hostSourceLister(ctx) {
|
|
413
|
+
export function hostSourceLister(ctx, { broker = null, brokerDetail = null, brokerState = null, timeoutMs = SOURCE_READ_TIMEOUT_MS } = {}) {
|
|
331
414
|
if (typeof ctx?.rpc !== "function") return null
|
|
332
|
-
|
|
415
|
+
const lister = async (source, input, sessionID, confirmedDigest = null) => {
|
|
333
416
|
const entry = sourceEntry(source)
|
|
334
417
|
if (!entry) throw new Error(`${oneLine(source, 40)} is not a source Memory knows`)
|
|
418
|
+
// A declared source is read through the contract's own `read`: the grant's
|
|
419
|
+
// rules, the descriptor's hash, the params and the page checks are the
|
|
420
|
+
// package's, and Memory keeps only its cursor.
|
|
421
|
+
if (entry.contract === "sources") {
|
|
422
|
+
return readSource({ entry, ctx, sessionID, account: input?.account ?? DEFAULT_ACCOUNT, scope: input?.scope, cursor: input?.cursor, limit: input?.limit, broker, brokerDetail, confirmedDigest, timeoutMs, readState: brokerState })
|
|
423
|
+
}
|
|
335
424
|
const client = ctx.rpc(rpcDefinition(entry))
|
|
336
425
|
if (typeof client?.[entry.tool] !== "function") throw new Error(`the host has no RPC method ${entry.rpc}/${entry.tool}`)
|
|
337
|
-
return readEventsAnswer(await client[entry.tool]({ sessionID, input: { ...input, ...entry.input } }), entry)
|
|
426
|
+
return readEventsAnswer(await client[entry.tool]({ sessionID, input: { ...input, ...entry.input } }), entry, input?.account ?? null)
|
|
427
|
+
}
|
|
428
|
+
// One broker deadline per tick: the collector resets it at the start of every run.
|
|
429
|
+
if (brokerState) {
|
|
430
|
+
lister.resetBrokerTick = () => {
|
|
431
|
+
brokerState.timedOut = false
|
|
432
|
+
brokerState.embedded?.clear?.()
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
return lister
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/** The host's own core RPC: jelly's `jelly.core`, which parks a plugin's ask and the host answers. */
|
|
439
|
+
export const CORE_RPC_ID = "jelly.core"
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* How long one source read gets. **Nothing bounded an embedded read:** a declared source's page was awaited
|
|
443
|
+
* through the host's tools RPC with no deadline at all (`readSource`'s `host.call`), so a hung plugin hung the
|
|
444
|
+
* collector. This is the budget proposed for both paths — a broker read and an embedded read of the same source
|
|
445
|
+
* — so a live broker that is merely slow (a 20 s provider read) is not mistaken for a silent one. 30 s is
|
|
446
|
+
* fifteen times the records client's own read timeout (`config.timeoutMs`, 2,000 ms) and well inside a
|
|
447
|
+
* 30-minute cadence.
|
|
448
|
+
*/
|
|
449
|
+
export const SOURCE_READ_TIMEOUT_MS = 30_000
|
|
450
|
+
/** The broker's default deadline: the same budget every embedded read of that source now gets. */
|
|
451
|
+
export const BROKER_TIMEOUT_MS = SOURCE_READ_TIMEOUT_MS
|
|
452
|
+
/** The deadline's own value, distinct from any answer a broker could give. */
|
|
453
|
+
const TIMEOUT = Symbol("broker timeout")
|
|
454
|
+
|
|
455
|
+
/**
|
|
456
|
+
* A transport to the host's source broker (`jelly.core.request`, kind `broker`),
|
|
457
|
+
* or null when the host gives plugins no tools RPC channel.
|
|
458
|
+
*
|
|
459
|
+
* **Never a bypass (B1).** The contract's `sourceReader` reads embedded when a
|
|
460
|
+
* transport throws or answers something that is not a broker answer, and that
|
|
461
|
+
* would read around jelly's grant and declaration checks. So this transport
|
|
462
|
+
* never throws and never answers nothing: an unreachable core plugin, a rejected
|
|
463
|
+
* call, an empty or malformed answer all become a refusal (`offered: true,
|
|
464
|
+
* ok: false, UNAVAILABLE`), which holds the binding. Only jelly's own explicit
|
|
465
|
+
* `{ offered: false }` (its broker is off) lets the read run embedded.
|
|
466
|
+
*
|
|
467
|
+
* **The capability is the resource (B3).** The broker is offered only where the
|
|
468
|
+
* host offers the `memory` resource (resource-token contract §1/§6), checked per
|
|
469
|
+
* read; every other host reads embedded and its RPC is never asked. A host that
|
|
470
|
+
* does offer it keeps the strict rule: only its explicit `{ offered: false }`
|
|
471
|
+
* reads embedded, and every failure — `rpc.unavailable`, `rpc.method_not_found`
|
|
472
|
+
* and any other — fails closed. No host identity is read.
|
|
473
|
+
* The package's `broker-client` still says never to *find* a broker by asking
|
|
474
|
+
* an RPC id: an RPC id has no owner, so another plugin could answer in the
|
|
475
|
+
* broker's name. The id is the source-contract's own.
|
|
476
|
+
*
|
|
477
|
+
* **A broker that never answers cannot stall collection (round 6).** The call is
|
|
478
|
+
* bounded by `timeoutMs` (default `SOURCE_READ_TIMEOUT_MS`, the same 30 s an
|
|
479
|
+
* embedded read gets, or the caller's), aborted when the client honours
|
|
480
|
+
* a signal, and turned into a refusal (`offered: true, ok: false, UNAVAILABLE`):
|
|
481
|
+
* the binding is held and never read embedded. Older Jelly betas, which answer
|
|
482
|
+
* nothing while their broker is off, therefore pause one binding instead of
|
|
483
|
+
* hanging the run; the next cadence tries again.
|
|
484
|
+
*
|
|
485
|
+
* @param {any} ctx the plugin context
|
|
486
|
+
* @param {{ detail?: object | null, timeoutMs?: number }} [host] the caller's deadline and detail
|
|
487
|
+
* @returns {null | ((request: object, signal?: AbortSignal) => Promise<any>)}
|
|
488
|
+
*/
|
|
489
|
+
export function hostBrokerTransport(ctx, { detail = null, timeoutMs = SOURCE_READ_TIMEOUT_MS, state = null } = {}) {
|
|
490
|
+
if (typeof ctx?.rpc !== "function") return null
|
|
491
|
+
const refuse = (message) => ({ offered: true, ok: false, code: "UNAVAILABLE", message, next: "Retry on the next cadence; nothing was read." })
|
|
492
|
+
return async (request, signal = undefined) => {
|
|
493
|
+
// The interim contract fact, asked per read: only a host that offers the
|
|
494
|
+
// `memory` resource can have a broker. Plain OpenCode hands a callable
|
|
495
|
+
// `rpc` too, and answers an unknown id with `rpc.unavailable`
|
|
496
|
+
// (`packages/core/src/rpc.ts:108-114`), so its channel must never be asked.
|
|
497
|
+
// A host that does offer the resource keeps the rule below: only its own
|
|
498
|
+
// `{ offered: false }` lets a read run embedded; every failure fails closed.
|
|
499
|
+
if (!memoryResourceAvailable()) return { offered: false }
|
|
500
|
+
// One deadline per tick (N7b): after the first broker timeout this run refuses the rest at once, so a
|
|
501
|
+
// host whose broker never answers costs one deadline, not one per binding. The next tick tries again.
|
|
502
|
+
if (state && state.timedOut === true) {
|
|
503
|
+
return refuse("jelly's broker did not answer earlier in this run, so nothing was read; the binding is retried on its next cadence")
|
|
504
|
+
}
|
|
505
|
+
let settled
|
|
506
|
+
const controller = new AbortController()
|
|
507
|
+
const onAbort = () => controller.abort()
|
|
508
|
+
signal?.addEventListener?.("abort", onAbort, { once: true })
|
|
509
|
+
let timer
|
|
510
|
+
const deadline = new Promise((resolve) => {
|
|
511
|
+
timer = setTimeout(() => (controller.abort(TIMEOUT), resolve(TIMEOUT)), timeoutMs)
|
|
512
|
+
})
|
|
513
|
+
try {
|
|
514
|
+
const client = ctx.rpc({ id: CORE_RPC_ID, events: {}, methods: { request: { input: { type: "object" }, output: { type: "object" } } } })
|
|
515
|
+
if (typeof client?.request !== "function") return refuse("jelly's core plugin is not answering, so its broker cannot be asked")
|
|
516
|
+
// The core plugin settles the parked call with the host's answer (`{answer, seal}`); a client that honours
|
|
517
|
+
// the signal aborts the parked request, so no call is left dangling.
|
|
518
|
+
// OpenCode's Promise adapter reads `options.signal` (`@opencode/plugin` 2.2.6
|
|
519
|
+
// `dist/promise/adapter.js:52-57`: `(input, options) => run(method(input), { signal: options?.signal })`),
|
|
520
|
+
// so the abort must travel as `{ signal }`; a bare signal is ignored and the call stays parked.
|
|
521
|
+
const call = client.request({ id: `mem-broker-${createHash("sha256").update(JSON.stringify(request)).digest("hex").slice(0, 16)}-${Date.now()}`, kind: "broker", body: request, keep: false }, { signal: controller.signal })
|
|
522
|
+
const raced = await Promise.race([call, deadline])
|
|
523
|
+
if (raced === TIMEOUT) {
|
|
524
|
+
if (state) state.timedOut = true
|
|
525
|
+
return refuse("jelly's broker did not answer in time, so nothing was read; the binding is retried on its next cadence")
|
|
526
|
+
}
|
|
527
|
+
settled = raced
|
|
528
|
+
} catch (error) {
|
|
529
|
+
if (controller.signal.aborted && controller.signal.reason === TIMEOUT) {
|
|
530
|
+
if (state) state.timedOut = true
|
|
531
|
+
return refuse("jelly's broker did not answer in time, so nothing was read; the binding is retried on its next cadence")
|
|
532
|
+
}
|
|
533
|
+
// A crashed or restarted core plugin, a missing method, or any other
|
|
534
|
+
// host error: a refusal, never an embedded read. On a host that offers
|
|
535
|
+
// `memory`, only its explicit `{ offered: false }` reads embedded.
|
|
536
|
+
return refuse(`jelly's broker could not be asked: ${oneLine(errorMessage(error), 200)}`)
|
|
537
|
+
} finally {
|
|
538
|
+
clearTimeout(timer)
|
|
539
|
+
signal?.removeEventListener?.("abort", onAbort)
|
|
540
|
+
}
|
|
541
|
+
const answer = settled?.answer
|
|
542
|
+
if (!isBrokerAnswer(answer)) return refuse(`jelly's broker answered nothing the contract defines`)
|
|
543
|
+
// The original code, before the package aliases it (NOT_GRANTED is a withdrawn jelly grant, not a
|
|
544
|
+
// changed declaration). The classifier uses this, never the message.
|
|
545
|
+
if (detail && answer.offered === true && answer.ok === false && typeof answer.code === "string") detail.code = answer.code
|
|
546
|
+
return answer
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* One page of a declared source, read through the contract's `read`.
|
|
552
|
+
*
|
|
553
|
+
* The host it is given is the binding's own session (its rules were made for
|
|
554
|
+
* this source and account), and the call is the plugin's tools-RPC method. The
|
|
555
|
+
* grant is Memory's, for the account and the scope the person confirmed, and
|
|
556
|
+
* for the descriptor hash of the install on disk: a changed descriptor is
|
|
557
|
+
* `PERMISSION_DENIED`, never a silent read.
|
|
558
|
+
*
|
|
559
|
+
* @param {{ entry: object, ctx: any, sessionID: string, account: string, scope?: object, cursor?: string, limit?: number, now?: number }} input
|
|
560
|
+
* @returns {Promise<object>} the page, in the shape the collector stores
|
|
561
|
+
*/
|
|
562
|
+
export async function readSource({ entry, ctx, sessionID, account, scope, cursor, limit = PAGE_LIMIT, now = Date.now(), broker = null, brokerDetail = null, confirmedDigest = null, timeoutMs = SOURCE_READ_TIMEOUT_MS, readState = null }) {
|
|
563
|
+
if (entry?.contract !== "sources" || !entry.item) throw new Error(`${oneLine(String(entry?.source ?? "?"), 40)} is not a declared source`)
|
|
564
|
+
if (typeof ctx?.rpc !== "function") throw new Error("this host gives plugins no tools RPC, so a source cannot be read")
|
|
565
|
+
const methods = { [entry.tool]: { input: { type: "object" }, output: { type: "object" } } }
|
|
566
|
+
const host = {
|
|
567
|
+
async openSession() {
|
|
568
|
+
return { sessionID, async close() {} }
|
|
569
|
+
},
|
|
570
|
+
async call({ method, input }) {
|
|
571
|
+
const client = ctx.rpc({ id: entry.rpc, events: {}, methods })
|
|
572
|
+
if (typeof client?.[method] !== "function") {
|
|
573
|
+
// The one refusal D11 needs: the contract's probe method, which every
|
|
574
|
+
// declared source implements. Declaring `sources.probe.action` beside it
|
|
575
|
+
// is optional. No events are read, here or anywhere.
|
|
576
|
+
const error = new Error(
|
|
577
|
+
`${oneLine(String(entry.source), 40)} implements no ${oneLine(String(method), 20)} method: the contract's probe is how Memory reads who it is signed in as (D11). Declaring sources.probe.action is optional; implementing the method is not. Nothing was read. Next step: add the ${oneLine(String(method), 20)} method to ${oneLine(String(entry.package ?? "its plugin"), 40)}'s tools RPC, then bind the source again.`,
|
|
578
|
+
)
|
|
579
|
+
error.code = "SCHEMA_VIOLATION"
|
|
580
|
+
throw error
|
|
581
|
+
}
|
|
582
|
+
// The embedded read’s budget, the same one the broker gets, and per plugin per tick: once one plugin’s
|
|
583
|
+
// read times out, its other bindings this run are held at once, and another plugin is unaffected (N8b).
|
|
584
|
+
const embedded = readState ? (readState.embedded ??= new Set()) : null
|
|
585
|
+
if (embedded?.has(entry.package)) {
|
|
586
|
+
const error = new Error("UNAVAILABLE: this source's plugin did not answer earlier in this run, so nothing was read. Next step: retry on the next cadence.")
|
|
587
|
+
error.code = "UNAVAILABLE"
|
|
588
|
+
throw error
|
|
589
|
+
}
|
|
590
|
+
try {
|
|
591
|
+
return await boundedSourceCall(client, method, input, sessionID, timeoutMs)
|
|
592
|
+
} catch (error) {
|
|
593
|
+
if (error?.timedOut === true) embedded?.add(entry.package)
|
|
594
|
+
throw error
|
|
595
|
+
}
|
|
596
|
+
},
|
|
597
|
+
}
|
|
598
|
+
const confirmed = isPlainObject(scope) && Object.keys(scope).length ? scope : undefined
|
|
599
|
+
// The digest the **person confirmed** (the consent's), never merely the one installed now: jelly's broker
|
|
600
|
+
// compares it with its own installed declaration and refuses a read under another one (B4).
|
|
601
|
+
const digest = typeof confirmedDigest === "string" && /^sha256:[0-9a-f]{64}$/.test(confirmedDigest) ? confirmedDigest : entry.descriptorHash
|
|
602
|
+
const installed = { package: entry.package, version: entry.version ?? "0.0.0", directory: "", sources: entry.descriptor, hash: digest }
|
|
603
|
+
const grant = { consumer: "memory", source: entry.source, account: account ?? DEFAULT_ACCOUNT, ...(confirmed ? { scope: confirmed } : {}), descriptorHash: digest }
|
|
604
|
+
// The package's reader: jelly's host broker first when it is offered (and the host gave one), else the
|
|
605
|
+
// embedded read. Its request carries `package`, `version`, `descriptorHash` and `consumer: "memory"`, so a
|
|
606
|
+
// broker whose installed declaration differs refuses instead of serving a read the person did not confirm.
|
|
607
|
+
const outcome = await sourceReader({ host, ...(broker ? { broker } : {}) }).read({
|
|
608
|
+
installed,
|
|
609
|
+
source: entry.source,
|
|
610
|
+
grant,
|
|
611
|
+
consumer: "memory",
|
|
612
|
+
...(confirmed ? { scope: confirmed } : {}),
|
|
613
|
+
...(cursor ? { cursor } : {}),
|
|
614
|
+
limit,
|
|
615
|
+
context: { now, timeZone: "UTC" },
|
|
616
|
+
})
|
|
617
|
+
if (!outcome.ok) {
|
|
618
|
+
const error = new Error(outcome.failure.message)
|
|
619
|
+
error.code = outcome.failure.code
|
|
620
|
+
error.next = outcome.failure.next
|
|
621
|
+
// The classifier's facts: how the read was made, and the refusal's own code before aliasing.
|
|
622
|
+
error.broker = outcome.via === "broker"
|
|
623
|
+
error.brokerCode = brokerDetail?.code ?? null
|
|
624
|
+
throw error
|
|
625
|
+
}
|
|
626
|
+
const page = outcome.page
|
|
627
|
+
return {
|
|
628
|
+
source: entry.source,
|
|
629
|
+
pageSource: page.source,
|
|
630
|
+
account: page.account,
|
|
631
|
+
owner: page.owner ?? null,
|
|
632
|
+
events: page.events,
|
|
633
|
+
cursor: page.cursor ?? "",
|
|
634
|
+
more: page.more === true,
|
|
635
|
+
refused: Array.isArray(page.refused) ? page.refused : [],
|
|
636
|
+
failed: Array.isArray(page.failed) ? page.failed : [],
|
|
637
|
+
// The page's own echo of what it read: kept, so the store can refuse a page
|
|
638
|
+
// that read wider than the person confirmed (plugin_scope).
|
|
639
|
+
scope: isPlainObject(page.scope) ? page.scope : null,
|
|
640
|
+
gap: isPlainObject(page.gap) ? page.gap : null,
|
|
641
|
+
cost: isPlainObject(page.cost) ? page.cost : null,
|
|
642
|
+
setup: null,
|
|
338
643
|
}
|
|
339
644
|
}
|
|
340
645
|
|
|
646
|
+
/**
|
|
647
|
+
* One call of a plugin's tools RPC, inside the source's read budget, aborted in the shape OpenCode reads
|
|
648
|
+
* (`@opencode/plugin` 2.2.6 `dist/promise/adapter.js:52-57`: `(input, options) => run(method(input), { signal: options?.signal })`).
|
|
649
|
+
* A call that outlives the budget is `UNAVAILABLE`, and the binding holds.
|
|
650
|
+
*/
|
|
651
|
+
async function boundedSourceCall(client, method, input, sessionID, timeoutMs) {
|
|
652
|
+
const controller = new AbortController()
|
|
653
|
+
// "Timed out" is decided by OUR deadline, not by how the promise settled: a plugin that rejects the moment it
|
|
654
|
+
// is aborted (which OpenCode's adapter does) rejects before the deadline's own resolution is delivered, and
|
|
655
|
+
// that must still count for the per-plugin circuit.
|
|
656
|
+
let fired = false
|
|
657
|
+
let timer
|
|
658
|
+
const deadline = new Promise((resolve) => {
|
|
659
|
+
timer = setTimeout(() => {
|
|
660
|
+
fired = true
|
|
661
|
+
controller.abort("source read timeout")
|
|
662
|
+
resolve(TIMEOUT)
|
|
663
|
+
}, timeoutMs)
|
|
664
|
+
})
|
|
665
|
+
try {
|
|
666
|
+
const raced = await Promise.race([client[method]({ sessionID, input }, { signal: controller.signal }), deadline])
|
|
667
|
+
if (raced === TIMEOUT || fired) {
|
|
668
|
+
const error = new Error("UNAVAILABLE: the source did not answer within its read budget, so nothing was read. Next step: retry on the next cadence.")
|
|
669
|
+
error.code = "UNAVAILABLE"
|
|
670
|
+
error.timedOut = true
|
|
671
|
+
throw error
|
|
672
|
+
}
|
|
673
|
+
return raced
|
|
674
|
+
} catch (error) {
|
|
675
|
+
if (fired && (error?.code === undefined || error?.code === "UNAVAILABLE")) {
|
|
676
|
+
const timeout = new Error("UNAVAILABLE: the source did not answer within its read budget, so nothing was read. Next step: retry on the next cadence.")
|
|
677
|
+
timeout.code = "UNAVAILABLE"
|
|
678
|
+
timeout.timedOut = true
|
|
679
|
+
throw timeout
|
|
680
|
+
}
|
|
681
|
+
throw error
|
|
682
|
+
} finally {
|
|
683
|
+
clearTimeout(timer)
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* One `probe` of a declared source, through the contract (D11): the source's own
|
|
689
|
+
* probe method, in the binding's session, answers the signed-in account and its
|
|
690
|
+
* owner. It reads no content, and it replaces the owner-only
|
|
691
|
+
* `events_list({ limit: 0 })` page for a source that has moved to the contract.
|
|
692
|
+
*
|
|
693
|
+
* The host is the binding's own session, as a page read's is: its rules allow
|
|
694
|
+
* the source's events action, its operations and its probe action.
|
|
695
|
+
*
|
|
696
|
+
* @param {{ entry: object, ctx: any, sessionID: string, account: string }} input
|
|
697
|
+
* @returns {Promise<{ account: string, owner: object, scopes?: string[] }>}
|
|
698
|
+
*/
|
|
699
|
+
export async function probeSource({ entry, ctx, sessionID, account, timeoutMs = SOURCE_READ_TIMEOUT_MS }) {
|
|
700
|
+
if (entry?.contract !== "sources" || !entry.item) throw new Error(`${oneLine(String(entry?.source ?? "?"), 40)} is not a declared source`)
|
|
701
|
+
// D11 reads the owner identity through the contract's `probe` and nothing
|
|
702
|
+
// else; the contract refuses `events_list({ limit: 0 })` for a declared
|
|
703
|
+
// source, so there is no page to fall back to (contract-v010 review, F5).
|
|
704
|
+
// `sources.probe` is **optional** in contract v0.1.0: its `probeRules` fences
|
|
705
|
+
// the probe action with `probeAction ?? methodAction(source)`, and this build's
|
|
706
|
+
// `sourceAllowRules` falls back the same way, so a plugin that implements the
|
|
707
|
+
// probe *method* without declaring the action is conformant and is probed. What
|
|
708
|
+
// is refused is a source with no probe method at all (the check in `call`).
|
|
709
|
+
if (typeof ctx?.rpc !== "function") throw new Error("this host gives plugins no tools RPC, so a source cannot be probed")
|
|
710
|
+
const methods = { probe: { input: { type: "object" }, output: { type: "object" } } }
|
|
711
|
+
/** The one refusal D11 needs: the contract's probe method, which every declared source implements. */
|
|
712
|
+
const missingMethod = (method) => {
|
|
713
|
+
// Declaring `sources.probe.action` beside the method is optional. No events
|
|
714
|
+
// are read, here or anywhere.
|
|
715
|
+
const error = new Error(
|
|
716
|
+
`${oneLine(String(entry.source), 40)} implements no ${oneLine(String(method), 20)} method: the contract's probe is how Memory reads who it is signed in as (D11). Declaring sources.probe.action is optional; implementing the method is not. Nothing was read. Next step: add the ${oneLine(String(method), 20)} method to ${oneLine(String(entry.package ?? "its plugin"), 40)}'s tools RPC, then bind the source again.`,
|
|
717
|
+
)
|
|
718
|
+
error.code = "SCHEMA_VIOLATION"
|
|
719
|
+
return error
|
|
720
|
+
}
|
|
721
|
+
const host = {
|
|
722
|
+
async openSession() {
|
|
723
|
+
return { sessionID, async close() {} }
|
|
724
|
+
},
|
|
725
|
+
async call({ method, input }) {
|
|
726
|
+
const client = ctx.rpc({ id: entry.rpc, events: {}, methods })
|
|
727
|
+
// Memory's own definition always carries the method, so a real host has it:
|
|
728
|
+
// the engine's own type is what a plugin without it answers (below).
|
|
729
|
+
if (typeof client?.[method] !== "function") throw missingMethod(method)
|
|
730
|
+
try {
|
|
731
|
+
return await boundedSourceCall(client, method, input, sessionID, timeoutMs)
|
|
732
|
+
} catch (error) {
|
|
733
|
+
if (error?.type === "rpc.method_not_found") throw missingMethod(method)
|
|
734
|
+
throw error
|
|
735
|
+
}
|
|
736
|
+
},
|
|
737
|
+
}
|
|
738
|
+
const installed = { package: entry.package, version: entry.version ?? "0.0.0", directory: "", sources: entry.descriptor, hash: entry.descriptorHash }
|
|
739
|
+
const outcome = await probe(host, { installed, source: entry.source, account: account ?? DEFAULT_ACCOUNT, identityOperations: ["probe"] })
|
|
740
|
+
if (!outcome.ok) {
|
|
741
|
+
const error = new Error(outcome.failure.message)
|
|
742
|
+
error.code = outcome.failure.code
|
|
743
|
+
error.next = outcome.failure.next
|
|
744
|
+
throw error
|
|
745
|
+
}
|
|
746
|
+
// The contract's one probe check, called by the consumer too: the answer names the source and the account,
|
|
747
|
+
// carries owner { id, label } (never null) and trust: "untrusted", and reads no content (D8, D230).
|
|
748
|
+
const problems = probeResultProblems(outcome.answer, { ...(entry.pageSource ? { source: entry.pageSource } : {}), account: account ?? DEFAULT_ACCOUNT })
|
|
749
|
+
if (problems.length > 0) {
|
|
750
|
+
const error = new Error(`SCHEMA_VIOLATION: ${entry.source}'s probe answer is not the contract's: ${problems[0]} Next step: report it to the plugin's maintainers.`)
|
|
751
|
+
error.code = "SCHEMA_VIOLATION"
|
|
752
|
+
throw error
|
|
753
|
+
}
|
|
754
|
+
return outcome.answer
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
/** A declared source's probe through the host's tools RPC; null when the host has no RPC client. */
|
|
758
|
+
export function hostSourceProber(ctx) {
|
|
759
|
+
if (typeof ctx?.rpc !== "function") return null
|
|
760
|
+
return async (source, input, sessionID) => {
|
|
761
|
+
const entry = sourceEntry(source)
|
|
762
|
+
if (!entry) throw new Error(`${oneLine(source, 40)} is not a source Memory knows`)
|
|
763
|
+
if (entry.contract !== "sources") throw new Error(`${oneLine(source, 40)} is read through its events_list, not probed`)
|
|
764
|
+
return probeSource({ entry, ctx, sessionID, account: input?.account ?? DEFAULT_ACCOUNT })
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
/**
|
|
769
|
+
* One refused event, kept in the run's own words (DESIGN-jelly-sources §2.3.1),
|
|
770
|
+
* because a count is not an account: the contract's policy advances the cursor
|
|
771
|
+
* past `refused[]` (`cursorAdvance`), so the id and the reason travel with the
|
|
772
|
+
* result to `mem_spaces` health and are never only a number.
|
|
773
|
+
*/
|
|
774
|
+
function noteRefusal(result, refusal) {
|
|
775
|
+
if (!isPlainObject(refusal)) return
|
|
776
|
+
const reason = refusal.reason ?? refusal.detail
|
|
777
|
+
// The reason is the source's own words, and it is kept in the result, in the
|
|
778
|
+
// status file and in health: redacted where it is kept, never only where it
|
|
779
|
+
// is shown (0.10.4's gate; review F2).
|
|
780
|
+
result.refused_reason = oneLine(errorMessage(typeof reason === "string" && reason ? reason : "the source refused an event"), 200)
|
|
781
|
+
const id = refusal.external_id === undefined || refusal.external_id === null ? null : oneLine(String(refusal.external_id), 100)
|
|
782
|
+
result.refused_id = id
|
|
783
|
+
}
|
|
784
|
+
|
|
341
785
|
/**
|
|
342
786
|
* The collector's core: schedule upkeep, one collection, and the owners a
|
|
343
787
|
* refused ingest reported.
|
|
344
788
|
*
|
|
345
|
-
* @param {{ scheduler?: { list(): Promise<any[]>, create(input: any): Promise<any>, remove(id: string): Promise<void> } | null, lister?: ReturnType<typeof hostSourceLister>, log?: any }} [input]
|
|
789
|
+
* @param {{ scheduler?: { list(): Promise<any[]>, create(input: any): Promise<any>, remove(id: string): Promise<void> } | null, lister?: ReturnType<typeof hostSourceLister>, prober?: ReturnType<typeof hostSourceProber>, log?: any }} [input]
|
|
346
790
|
*/
|
|
347
|
-
export function createCollector({ scheduler = null, lister = null, log } = {}) {
|
|
791
|
+
export function createCollector({ scheduler = null, lister = null, prober = null, log } = {}) {
|
|
348
792
|
const reported = new Map()
|
|
349
793
|
|
|
350
794
|
function noteOwner(source, account, owner) {
|
|
@@ -408,7 +852,9 @@ export function createCollector({ scheduler = null, lister = null, log } = {}) {
|
|
|
408
852
|
* due?: (binding: { source: string, account: string }) => boolean,
|
|
409
853
|
* }} input
|
|
410
854
|
*/
|
|
411
|
-
async function runOnce({ sources, only, cursor, ingest, sessionID, results = [], due = () => true }) {
|
|
855
|
+
async function runOnce({ sources, only, cursor, ingest, sessionID, results = [], due = () => true, pause = null }) {
|
|
856
|
+
// One broker deadline per tick (N7b): a broker that timed out this run does not cost each later binding.
|
|
857
|
+
lister?.resetBrokerTick?.()
|
|
412
858
|
for (const binding of collectableSources(sources, { only })) {
|
|
413
859
|
if (!due(binding)) {
|
|
414
860
|
results.push({ ...binding, pages: 0, accepted: 0, duplicates: 0, refused: 0, more: false, notDue: true, everyMinutes: cadenceMs(binding.source) / 60_000, done: true })
|
|
@@ -428,29 +874,162 @@ export function createCollector({ scheduler = null, lister = null, log } = {}) {
|
|
|
428
874
|
continue
|
|
429
875
|
}
|
|
430
876
|
let at = typeof state.cursor === "string" && state.cursor ? state.cursor : undefined
|
|
877
|
+
const scope = isPlainObject(state.scope) ? state.scope : {}
|
|
878
|
+
// The owner the person confirmed: a page that answers another is a rebind
|
|
879
|
+
// (§15.3), paused here with both labels — never read under the new one.
|
|
880
|
+
const bound = isPlainObject(state.owner) && typeof state.owner.id === "string" && state.owner.id ? state.owner : null
|
|
881
|
+
// The catalog entry this binding reads under: a declared source's page is
|
|
882
|
+
// ingested under the binding's own account (G2), a legacy one as before.
|
|
883
|
+
const boundEntry = sourceEntry(binding.source)
|
|
884
|
+
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
885
|
+
try {
|
|
431
886
|
for (let page = 0; page < PAGES_PER_SOURCE; page += 1) {
|
|
432
|
-
const answer = await lister(binding.source, listInput(binding, at), session)
|
|
887
|
+
const answer = await lister(binding.source, listInput(binding, at, scope), session, isPlainObject(state) ? state.digest ?? null : null)
|
|
433
888
|
if (!answer || !answer.cursor) {
|
|
434
889
|
result.error = "the source's events_list gave no events and cursor"
|
|
435
890
|
break
|
|
436
891
|
}
|
|
437
|
-
|
|
892
|
+
if (bound && isPlainObject(answer.owner) && typeof answer.owner.id === "string" && answer.owner.id && answer.owner.id !== bound.id) {
|
|
893
|
+
const was = bound.label ?? bound.id
|
|
894
|
+
const now = answer.owner.label ?? answer.owner.id
|
|
895
|
+
const reason = `${binding.source} (${binding.account}) was bound to ${was}; the provider now answers ${now}. Next step: rebind it in Memory's console to confirm the new account.`
|
|
896
|
+
result.code = "OWNER_CHANGED"
|
|
897
|
+
result.action = "pause"
|
|
898
|
+
result.error = reason
|
|
899
|
+
if (typeof pause === "function") {
|
|
900
|
+
try {
|
|
901
|
+
await pause(binding, reason, "OWNER_CHANGED")
|
|
902
|
+
result.paused = true
|
|
903
|
+
} catch (error) {
|
|
904
|
+
result.pause_error = oneLine(errorMessage(error), 200)
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
break
|
|
908
|
+
}
|
|
909
|
+
// The answer's own account, and the binding's own source (§4.5): a
|
|
910
|
+
// declared source's page names the plugin's name for it (`slack`), while
|
|
911
|
+
// the person confirmed the descriptor's id (`slack.workspace`), and the
|
|
912
|
+
// store keys the consent and the cursor by that id — as a job does.
|
|
438
913
|
if (answer.setup) result.setup = answer.setup
|
|
439
|
-
|
|
914
|
+
// The contract's page rules (§8.3.4): `failed[]` holds the cursor back,
|
|
915
|
+
// `refused[]` does not, and a `gap` advances with the hole recorded.
|
|
916
|
+
const move = cursorAdvance(answer)
|
|
917
|
+
if (Array.isArray(answer.refused) && answer.refused.length) {
|
|
918
|
+
result.refused += answer.refused.length
|
|
919
|
+
noteRefusal(result, answer.refused.at(-1))
|
|
920
|
+
}
|
|
921
|
+
if (Array.isArray(answer.failed) && answer.failed.length) {
|
|
922
|
+
result.failed = answer.failed.length
|
|
923
|
+
// `held` is the page's code, or its own words when it sent none: it
|
|
924
|
+
// is kept in the run's result and the status file, so redacted where
|
|
925
|
+
// it is kept (0.11 review S1).
|
|
926
|
+
result.held = move.hidden === null || move.hidden === undefined ? move.hidden : oneLine(errorMessage(move.hidden), 100)
|
|
927
|
+
}
|
|
928
|
+
if (answer.gap) result.gap = answer.gap
|
|
929
|
+
// The page's own echo of what it read, when it gives one: the store
|
|
930
|
+
// refuses a page outside the person's confirmation (plugin_scope). A
|
|
931
|
+
// page that echoes nothing is sent the consented scope, which the read
|
|
932
|
+
// carried, so a scoped binding is not refused for saying nothing.
|
|
933
|
+
const echo = isPlainObject(answer.scope) ? answer.scope : null
|
|
934
|
+
// **The binding's account, for a declared source** (G2). A source with
|
|
935
|
+
// `accounts.named: false` reads one account, and the person's binding is
|
|
936
|
+
// keyed by the account its probe reported (`history.sessions` answers
|
|
937
|
+
// the machine it indexed, `linear.activity` a workspace user). The
|
|
938
|
+
// page's own account is the same one; sending it for a legacy source
|
|
939
|
+
// keeps that path as it was.
|
|
940
|
+
const saved = await ingest({
|
|
941
|
+
source: binding.source,
|
|
942
|
+
account: boundEntry?.contract === "sources" ? binding.account : answer.account,
|
|
943
|
+
owner: answer.owner,
|
|
944
|
+
events: answer.events,
|
|
945
|
+
...(echo && Object.keys(echo).length ? { scope: echo } : Object.keys(scope).length ? { scope } : {}),
|
|
946
|
+
// Held: the events are stored idempotently and the cursor does not move.
|
|
947
|
+
...(move.advance && answer.cursor ? { cursor: answer.cursor } : {}),
|
|
948
|
+
...(answer.gap ? { gap: answer.gap } : {}),
|
|
949
|
+
...(answer.cost ? { cost: answer.cost } : {}),
|
|
950
|
+
})
|
|
440
951
|
result.pages += 1
|
|
441
952
|
result.accepted += Number(saved?.accepted) || 0
|
|
442
953
|
result.duplicates += Number(saved?.duplicates) || 0
|
|
443
|
-
|
|
444
|
-
|
|
954
|
+
// The page's own `refused[]` and the store's are disjoint — the store
|
|
955
|
+
// is sent only the page's events — so both are added: every refused
|
|
956
|
+
// event is counted once (DESIGN-jelly-sources §2.3.1).
|
|
957
|
+
if (saved?.refused !== undefined) result.refused += Number(saved.refused) || 0
|
|
958
|
+
if (saved?.newest_refusal) noteRefusal(result, saved.newest_refusal)
|
|
959
|
+
if (saved?.resync === false) {
|
|
960
|
+
result.error = "the source restarted its window twice today; the binding waits until tomorrow rather than read again blind"
|
|
961
|
+
result.action = "pause"
|
|
962
|
+
break
|
|
963
|
+
}
|
|
964
|
+
if (move.advance && saved?.cursor_saved !== true) {
|
|
445
965
|
result.error = saved?.error || "Memory did not save the cursor, so collection stopped here"
|
|
446
966
|
break
|
|
447
967
|
}
|
|
968
|
+
if (!move.advance) break
|
|
448
969
|
at = answer.cursor
|
|
449
970
|
result.more = answer.more
|
|
450
971
|
if (!answer.more) break
|
|
451
972
|
}
|
|
973
|
+
break
|
|
974
|
+
} catch (error) {
|
|
975
|
+
// §8.3.4: a code is a pause, a hold or a reset. A reset reads once more
|
|
976
|
+
// from the window; a pause pauses the binding through its callback.
|
|
977
|
+
const failure = failureFromError(error)
|
|
978
|
+
// A PERMISSION_DENIED is decided by fact, never by its message: the broker's refusal is a changed
|
|
979
|
+
// declaration only when the digest installed now differs from the one the person confirmed. Equal
|
|
980
|
+
// digests mean the access itself was refused (a withdrawn grant, or the provider), which re-consenting
|
|
981
|
+
// cannot fix, so that state is `access_refused` with its own sentence. An embedded refusal is the
|
|
982
|
+
// source's own, and keeps `failureAction`'s meaning.
|
|
983
|
+
const entry = sourceEntry(binding.source)
|
|
984
|
+
const confirmed = isPlainObject(state) && typeof state.digest === "string" ? state.digest : null
|
|
985
|
+
const installed = typeof entry?.descriptorHash === "string" ? entry.descriptorHash : null
|
|
986
|
+
const viaBroker = error?.broker === true
|
|
987
|
+
const changed = failure.code === "PERMISSION_DENIED" && viaBroker && (!confirmed || !installed || installed !== confirmed)
|
|
988
|
+
const accessRefused = failure.code === "PERMISSION_DENIED" && viaBroker && !changed
|
|
989
|
+
const action = changed || accessRefused ? { action: "pause", reset: "none", resync: false } : failureAction(failure.code)
|
|
990
|
+
result.code = changed ? "DECLARATION_CHANGED" : accessRefused ? "ACCESS_REFUSED" : failure.code
|
|
991
|
+
const refusalMessage = "Memory's access to this source was refused. If you withdrew it, grant it again where you withdrew it."
|
|
992
|
+
result.error = changed ? "the source changed its declaration or its grant; confirm it again" : accessRefused ? refusalMessage : failure.message
|
|
993
|
+
result.action = action.action
|
|
994
|
+
if (action.action === "reset" && attempt === 1) {
|
|
995
|
+
at = undefined
|
|
996
|
+
result.reset = action.reset
|
|
997
|
+
// The read that asked for the reset is history: what this run reports
|
|
998
|
+
// as its outcome is the retry's own, not the code it recovered from.
|
|
999
|
+
result.code = undefined
|
|
1000
|
+
result.error = undefined
|
|
1001
|
+
result.action = undefined
|
|
1002
|
+
continue
|
|
1003
|
+
}
|
|
1004
|
+
if (action.action === "pause" && typeof pause === "function") {
|
|
1005
|
+
// The store holds the pause; this run only says whether it took.
|
|
1006
|
+
try {
|
|
1007
|
+
// An account that is not the bound one says so in the store: the person confirms the right account.
|
|
1008
|
+
const needsAccount = failure.code === "ACCOUNT_MISMATCH" || failure.code === "ACCOUNT_UNKNOWN"
|
|
1009
|
+
await pause(
|
|
1010
|
+
binding,
|
|
1011
|
+
changed
|
|
1012
|
+
? "the source changed its declaration or its grant; confirm it again"
|
|
1013
|
+
: accessRefused
|
|
1014
|
+
? "Memory's access to this source was refused. If you withdrew it, grant it again where you withdrew it."
|
|
1015
|
+
: needsAccount
|
|
1016
|
+
? action.reason
|
|
1017
|
+
: failure.message,
|
|
1018
|
+
changed ? "DECLARATION_CHANGED" : accessRefused ? "ACCESS_REFUSED" : failure.code,
|
|
1019
|
+
)
|
|
1020
|
+
result.paused = true
|
|
1021
|
+
} catch (error) {
|
|
1022
|
+
result.pause_error = oneLine(errorMessage(error), 200)
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
break
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
452
1028
|
} catch (error) {
|
|
453
|
-
|
|
1029
|
+
const failure = failureFromError(error)
|
|
1030
|
+
result.error = failure.message
|
|
1031
|
+
result.code = failure.code
|
|
1032
|
+
result.action = failureAction(failure.code).action
|
|
454
1033
|
} finally {
|
|
455
1034
|
result.done = true
|
|
456
1035
|
}
|
|
@@ -483,14 +1062,29 @@ export function createCollector({ scheduler = null, lister = null, log } = {}) {
|
|
|
483
1062
|
}
|
|
484
1063
|
let body
|
|
485
1064
|
try {
|
|
486
|
-
|
|
487
|
-
|
|
1065
|
+
// A declared source has the contract's `probe` (D11); the owner-only
|
|
1066
|
+
// `events_list({ limit: 0 })` page is the previous contract's.
|
|
1067
|
+
const answer = entry.contract === "sources"
|
|
1068
|
+
? await (prober ? prober(asked.source, { account: asked.account }, session) : Promise.reject(new Error("this host gives plugins no tools RPC, so a source cannot be probed")))
|
|
1069
|
+
: await (lister ? lister(asked.source, probeInput(entry, asked.account), session) : Promise.reject(new Error("this host gives plugins no way to call another plugin's events_list")))
|
|
488
1070
|
if (answer?.owner?.id) {
|
|
489
1071
|
const owner = { id: oneLine(answer.owner.id, 300), label: oneLine(answer.owner.label ?? answer.owner.email ?? answer.owner.id, 300) }
|
|
490
|
-
body = {
|
|
1072
|
+
body = {
|
|
1073
|
+
source: asked.source,
|
|
1074
|
+
account: asked.account,
|
|
1075
|
+
owner,
|
|
1076
|
+
reported_account: answer.account,
|
|
1077
|
+
// The installed declaration's digest, read from disk: a re-consent records the digest the installed
|
|
1078
|
+
// package has now, so jelly never refuses a read made under the old one.
|
|
1079
|
+
...(entry.descriptorHash ? { digest: entry.descriptorHash } : {}),
|
|
1080
|
+
// The note is the source's or the plugin's own sentence: it is
|
|
1081
|
+
// reported to the store (`plugin_requests.note`) and shown from it,
|
|
1082
|
+
// so it is redacted here, where it is written (review F1/F2).
|
|
1083
|
+
...(answer.setup || entry.setupOptions ? { note: oneLine(errorMessage(answer.setup || entry.setupOptions), 400) } : {}),
|
|
1084
|
+
}
|
|
491
1085
|
probe.owner = owner
|
|
492
1086
|
} else {
|
|
493
|
-
body = { source: asked.source, account: asked.account, note: answer?.setup || `${entry.package}'s events_list named no signed-in account; sign in to it in its own plugin.` }
|
|
1087
|
+
body = { source: asked.source, account: asked.account, note: oneLine(errorMessage(answer?.setup || `${entry.package}'s events_list named no signed-in account; sign in to it in its own plugin.`), 400) }
|
|
494
1088
|
}
|
|
495
1089
|
} catch (error) {
|
|
496
1090
|
body = { source: asked.source, account: asked.account, note: oneLine(errorMessage(error), 400) }
|