@joe-broadhead/opencode-mem 0.9.14 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1856 -0
- package/README.md +15 -7
- package/SECURITY.md +3 -1
- package/docs/agent-instructions.md +1 -1
- package/docs/agent-protocol.md +7 -7
- package/docs/permissions.md +16 -0
- package/package.json +8921 -1070
- package/shared/page-paths.js +44 -0
- package/skills/mem.pages/SKILL.md +3 -1
- package/skills/mem.usage/SKILL.md +15 -15
- package/src/action-grant-keys.js +159 -0
- package/src/action-grant.js +439 -0
- package/src/actions.js +1690 -0
- package/src/backend/records.js +25 -2
- package/src/backend/v3.js +76 -22
- package/src/call-shapes.js +17 -16
- package/src/collector-lock.js +533 -62
- package/src/collector-service.js +153 -26
- package/src/collector.js +623 -30
- package/src/config.js +88 -7
- package/src/connect.js +106 -8
- package/src/credentials.js +123 -25
- package/src/events.js +174 -3
- package/src/forget-turn.js +141 -0
- package/src/host-refusal.js +78 -0
- package/src/index.js +209 -22
- package/src/jobs/host.js +557 -0
- package/src/ops/codes.js +110 -0
- package/src/ops/copy-lineage.js +42 -0
- package/src/ops/index.js +28 -0
- package/src/ops/space-ops.js +1936 -0
- package/src/outlinks.js +103 -0
- package/src/p2-reservation.js +151 -0
- package/src/permissions.js +30 -0
- package/src/plugins/descriptor.js +62 -0
- package/src/plugins/discovery.js +318 -0
- package/src/plugins/grants.js +178 -0
- package/src/plugins/locate.js +319 -0
- package/src/plugins/manifest.js +1162 -0
- package/src/plugins/source-contract/index.js +34 -0
- package/src/plugins/source-contract/policy.js +102 -0
- package/src/plugins/source-contract/vendor/PROVENANCE.json +17 -0
- package/src/plugins/source-contract/vendor/RELEASE-SHA256SUMS +3 -0
- package/src/plugins/source-contract/vendor/RELEASE.json +17 -0
- package/src/plugins/source-contract/vendor/SHA256SUMS +41 -0
- package/src/plugins/source-contract/vendor/THIRD-PARTY-NOTICES.md +35 -0
- package/src/plugins/source-contract/vendor/broker-client.js +93 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-3M5PAKSS.js +84 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-4ERCA4TK.js +185 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-6HDIB2SA.js +183 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-ELPJD3ND.js +10 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-HLVAKAYG.js +10 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-I2RXJJJM.js +81 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-KWEWSZ4V.js +505 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-KYWBTAF4.js +182 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-Q3RQA5J2.js +249 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-RRUAIQ4I.js +156 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-RYNKNCPP.js +165 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-W5SHR7DQ.js +9477 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-ZUKF7P6U.js +653 -0
- package/src/plugins/source-contract/vendor/cursor.js +26 -0
- package/src/plugins/source-contract/vendor/descriptor.js +34 -0
- package/src/plugins/source-contract/vendor/events.js +257 -0
- package/src/plugins/source-contract/vendor/index.js +10 -0
- package/src/plugins/source-contract/vendor/params.js +22 -0
- package/src/plugins/source-contract/vendor/permission.js +28 -0
- package/src/plugins/source-contract/vendor/pick.js +22 -0
- package/src/plugins/source-contract/vendor/read.js +29 -0
- package/src/plugins/source-contract/vendor/registry.js +23 -0
- package/src/plugins/source-contract/vendor/schema.js +17 -0
- package/src/plugins/source-contract/vendor/schemas/output.agenda.json +96 -0
- package/src/plugins/source-contract/vendor/schemas/output.countdown.json +28 -0
- package/src/plugins/source-contract/vendor/schemas/output.image.json +28 -0
- package/src/plugins/source-contract/vendor/schemas/output.inbox.json +84 -0
- package/src/plugins/source-contract/vendor/schemas/output.links.json +46 -0
- package/src/plugins/source-contract/vendor/schemas/output.list.json +57 -0
- package/src/plugins/source-contract/vendor/schemas/output.metric.json +61 -0
- package/src/plugins/source-contract/vendor/schemas/output.note.json +14 -0
- package/src/plugins/source-contract/vendor/schemas/output.progress.json +50 -0
- package/src/plugins/source-contract/vendor/schemas/output.record.json +38 -0
- package/src/plugins/source-contract/vendor/schemas/output.table.json +61 -0
- package/src/plugins/source-contract/vendor/schemas/output.thread.json +62 -0
- package/src/plugins/source-contract/vendor/schemas/sources.json +819 -0
- package/src/plugins/source-contract/vendor/schemas.js +19 -0
- package/src/plugins/source-contract/vendor/tools-rpc.js +42 -0
- package/src/plugins/verify.js +56 -0
- package/src/protocol.js +11 -6
- package/src/recall.js +92 -5
- package/src/records.js +61 -2
- package/src/requests.js +216 -0
- package/src/runtime.js +89 -7
- package/src/setup-time.js +68 -0
- package/src/skills.js +230 -8
- package/src/source-ids.js +25 -0
- package/src/sources/declaration.js +365 -0
- package/src/sources/reads.js +772 -0
- package/src/sources/rpc.js +151 -0
- package/src/sources.js +389 -92
- package/src/spaces.js +283 -11
- package/src/spool.js +61 -4
- package/src/stated.js +10 -5
- package/src/storable.js +139 -0
- package/src/tools/commitments.js +74 -23
- package/src/tools/held-write.js +46 -19
- package/src/tools/index.js +28 -0
- package/src/tools/ingest.js +38 -2
- package/src/tools/memory.js +56 -25
- package/src/tools/page.js +212 -95
- package/src/tools/recall.js +218 -4
- package/src/tools/share.js +42 -3
- package/src/tools/shared.js +84 -6
- package/src/tools/space-lifecycle.js +278 -0
- package/src/tools/spaces.js +381 -30
- package/src/tools/status.js +9 -2
- package/src/tools/wiki.js +231 -116
- package/src/util.js +110 -6
- package/src/vendor/source-contract/README.md +24 -0
- package/src/vendor/source-contract/THIRD-PARTY-NOTICES.md +35 -0
- package/src/vendor/source-contract/VENDOR.json +81 -0
- package/src/vendor/source-contract/chunks/chunk-3M5PAKSS.js +84 -0
- package/src/vendor/source-contract/chunks/chunk-6HDIB2SA.js +183 -0
- package/src/vendor/source-contract/chunks/chunk-HLVAKAYG.js +10 -0
- package/src/vendor/source-contract/chunks/chunk-I2RXJJJM.js +81 -0
- package/src/vendor/source-contract/chunks/chunk-KWEWSZ4V.js +505 -0
- package/src/vendor/source-contract/chunks/chunk-W5SHR7DQ.js +9477 -0
- package/src/vendor/source-contract/chunks/chunk-ZUKF7P6U.js +653 -0
- package/src/vendor/source-contract/cursor.js +26 -0
- package/src/vendor/source-contract/events.js +257 -0
- package/src/vendor/source-contract/registry.js +23 -0
- package/src/vendor/source-contract/tools-rpc.js +42 -0
- package/src/views/access-reads.js +256 -0
- package/src/views/communities.js +77 -0
- package/src/views/contracts.js +1082 -23
- package/src/views/inbox-bytes.js +103 -0
- package/src/views/profile.js +31 -0
- package/src/views/receipt.js +26 -1
- package/src/views/records-reads.js +520 -41
- package/src/views/rpc.js +153 -25
- package/src/views/shape.js +8 -5
- package/src/views/write-state.js +89 -0
- package/src/wiki-ingest.js +215 -0
- package/src/wiki-state.js +228 -0
- package/views/pages.html +2 -2
package/src/collector.js
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { errorMessage, isPlainObject, oneLine } from "./util.js"
|
|
2
2
|
import { createHash } from "node:crypto"
|
|
3
|
-
import {
|
|
3
|
+
import { isSourceName } from "./source-ids.js"
|
|
4
|
+
import { ACCOUNT_NAME, DEFAULT_EVERY_MINUTES, contractRefusal, rpcDefinition, sourceAllowRules, sourceEntry } from "./sources.js"
|
|
5
|
+
import { acceptAnswer, probe, read } from "./plugins/source-contract/vendor/read.js"
|
|
6
|
+
import { isBrokerAnswer, sourceReader } from "./plugins/source-contract/vendor/broker-client.js"
|
|
7
|
+
import { probeResultProblems } from "./plugins/source-contract/vendor/events.js"
|
|
8
|
+
import { cursorAdvance, failureAction, failureFromError } from "./plugins/source-contract/policy.js"
|
|
9
|
+
import { memoryResourceAvailable } from "./credentials.js"
|
|
4
10
|
|
|
5
11
|
/**
|
|
6
12
|
* The memory collector (docs/design/memory-v1-contracts.md §4.4, §4.5).
|
|
@@ -29,9 +35,12 @@ import { ACCOUNT_NAME, DEFAULT_EVERY_MINUTES, SOURCE_REGISTRY, contractRefusal,
|
|
|
29
35
|
* that adds the API does not reach it either. `collectCheckCode` and
|
|
30
36
|
* `createCollector({scheduler})` stay as pure units; nothing wires a scheduler.
|
|
31
37
|
*
|
|
32
|
-
* Every `events_list` answer must carry `
|
|
33
|
-
*
|
|
34
|
-
*
|
|
38
|
+
* Every `events_list` answer must carry `account` (§4.5); one without it is
|
|
39
|
+
* refused with the release its plugin needs. The ingest is keyed by the
|
|
40
|
+
* binding's own source and the answer's account: a declared source's page names
|
|
41
|
+
* the plugin's name for it (`slack`), while the person confirmed the
|
|
42
|
+
* descriptor's id (`slack.workspace`), which is what the store holds the
|
|
43
|
+
* consent and the cursor under — the same key a job uses.
|
|
35
44
|
*/
|
|
36
45
|
|
|
37
46
|
export const COLLECT_SCHEDULE = "memory-collect"
|
|
@@ -108,13 +117,11 @@ export function sessionEffect(action, resource = "*", rules = []) {
|
|
|
108
117
|
return rules.findLast((rule) => wildcard(rule.action, action) && wildcard(rule.resource, resource))?.effect ?? "ask"
|
|
109
118
|
}
|
|
110
119
|
|
|
111
|
-
/**
|
|
112
|
-
const SOURCE_NAME = /^[a-z][a-z0-9_-]{0,63}$/
|
|
120
|
+
/** An identifier in a declaration or a template. */
|
|
113
121
|
const IDENTIFIER = /^[a-zA-Z][a-zA-Z0-9_]*$/
|
|
114
122
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
}
|
|
123
|
+
// A binding's source: a built-in name, or a declared source's contract id (src/source-ids.js).
|
|
124
|
+
export { isSourceName } from "./source-ids.js"
|
|
118
125
|
|
|
119
126
|
export function isAccountName(value) {
|
|
120
127
|
return typeof value === "string" && ACCOUNT_NAME.test(value)
|
|
@@ -141,7 +148,9 @@ export function collectableSources(sources, { only } = {}) {
|
|
|
141
148
|
if (entry.status !== "confirmed") continue
|
|
142
149
|
const source = String(entry.source ?? "")
|
|
143
150
|
const account = String(entry.account ?? "") || DEFAULT_ACCOUNT
|
|
144
|
-
|
|
151
|
+
const known = sourceEntry(source)
|
|
152
|
+
// A snapshot source is read on demand (`snapshot_read`), never collected as events.
|
|
153
|
+
if (!known || known.kind !== "events" || !isAccountName(account)) continue
|
|
145
154
|
if (only && source !== only) continue
|
|
146
155
|
const binding = { source, account }
|
|
147
156
|
seen.set(bindingKey(binding), binding)
|
|
@@ -160,13 +169,23 @@ function accountInput(entry, account) {
|
|
|
160
169
|
return named !== DEFAULT_ACCOUNT || entry?.namesAccount ? { account: named } : {}
|
|
161
170
|
}
|
|
162
171
|
|
|
163
|
-
|
|
172
|
+
/**
|
|
173
|
+
* One page's input: the cursor, the account, the source's own input, and **the
|
|
174
|
+
* scope the person confirmed**. A scoped source that is not told its scope reads
|
|
175
|
+
* its plugin's whole option, and its page is then refused as out of scope.
|
|
176
|
+
*
|
|
177
|
+
* @param {{ source: string, account?: string }} binding
|
|
178
|
+
* @param {string | undefined} cursor
|
|
179
|
+
* @param {Record<string, string[]>} [scope] the consented scope (`events.cursor` returns it)
|
|
180
|
+
*/
|
|
181
|
+
export function listInput(binding, cursor, scope) {
|
|
164
182
|
const entry = sourceEntry(binding.source)
|
|
165
183
|
return {
|
|
166
184
|
...(cursor ? { cursor } : {}),
|
|
167
185
|
limit: PAGE_LIMIT,
|
|
168
186
|
...accountInput(entry, binding.account),
|
|
169
187
|
...(entry?.input ?? {}),
|
|
188
|
+
...(isPlainObject(scope) && Object.keys(scope).length ? { scope } : {}),
|
|
170
189
|
}
|
|
171
190
|
}
|
|
172
191
|
|
|
@@ -244,7 +263,8 @@ export class SourceContractError extends Error {
|
|
|
244
263
|
* entry's) and `account`. Throws a SourceContractError otherwise; null when it
|
|
245
264
|
* is not an answer at all.
|
|
246
265
|
*/
|
|
247
|
-
export function readEventsAnswer(value, entry) {
|
|
266
|
+
export function readEventsAnswer(value, entry, boundAccount = null) {
|
|
267
|
+
if (entry?.contract === "sources") return readDeclaredAnswer(value, entry, boundAccount)
|
|
248
268
|
let answer = value
|
|
249
269
|
// A wrapper's text beside its output: a source's own words, such as Slack's "No Slack channel is enabled".
|
|
250
270
|
const said = isPlainObject(answer) && isPlainObject(answer.output) && typeof (answer.content ?? answer.text) === "string" ? String(answer.content ?? answer.text) : ""
|
|
@@ -274,6 +294,59 @@ export function readEventsAnswer(value, entry) {
|
|
|
274
294
|
}
|
|
275
295
|
}
|
|
276
296
|
|
|
297
|
+
/**
|
|
298
|
+
* A declared source's answer, checked by the contract's own `acceptAnswer`: the
|
|
299
|
+
* page's source and account, `trust: "untrusted"`, every event's shape, the
|
|
300
|
+
* size against its `maxBytes`, and `refused`/`failed`/`gap`/`cost`. A failure
|
|
301
|
+
* carries the contract's code, so the collector can act on it (§8.3.4).
|
|
302
|
+
*/
|
|
303
|
+
function readDeclaredAnswer(value, entry, account) {
|
|
304
|
+
const outcome = acceptAnswer(entry.item, account || DEFAULT_ACCOUNT, unwrapResult(value))
|
|
305
|
+
if (!outcome.ok) {
|
|
306
|
+
const error = new Error(`${outcome.failure.code}: ${outcome.failure.message}`)
|
|
307
|
+
error.code = outcome.failure.code
|
|
308
|
+
error.next = outcome.failure.next
|
|
309
|
+
throw error
|
|
310
|
+
}
|
|
311
|
+
const page = outcome.page
|
|
312
|
+
return {
|
|
313
|
+
// The binding's id (`gws.gmail`), never the plugin's short page name (`gmail`).
|
|
314
|
+
source: entry.source,
|
|
315
|
+
pageSource: page.source,
|
|
316
|
+
account: page.account,
|
|
317
|
+
owner: page.owner ?? null,
|
|
318
|
+
events: page.events,
|
|
319
|
+
cursor: page.cursor ?? "",
|
|
320
|
+
more: page.more === true,
|
|
321
|
+
refused: Array.isArray(page.refused) ? page.refused : [],
|
|
322
|
+
failed: Array.isArray(page.failed) ? page.failed : [],
|
|
323
|
+
// The page's own echo of what it read: kept, so the store can refuse a page
|
|
324
|
+
// that read wider than the person confirmed (plugin_scope).
|
|
325
|
+
scope: isPlainObject(page.scope) ? page.scope : null,
|
|
326
|
+
gap: isPlainObject(page.gap) ? page.gap : null,
|
|
327
|
+
cost: isPlainObject(page.cost) ? page.cost : null,
|
|
328
|
+
setup: null,
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** The payload of an RPC result: its `output`, its structured content, or the JSON in its text. */
|
|
333
|
+
function unwrapResult(value) {
|
|
334
|
+
let answer = value
|
|
335
|
+
if (isPlainObject(answer) && !Array.isArray(answer.events)) {
|
|
336
|
+
if (isPlainObject(answer.output)) answer = answer.output
|
|
337
|
+
else if (isPlainObject(answer.structuredContent)) answer = answer.structuredContent
|
|
338
|
+
else if (typeof (answer.content ?? answer.text) === "string") answer = answer.content ?? answer.text
|
|
339
|
+
}
|
|
340
|
+
if (typeof answer === "string") {
|
|
341
|
+
try {
|
|
342
|
+
answer = JSON.parse(answer)
|
|
343
|
+
} catch {
|
|
344
|
+
return value
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
return answer
|
|
348
|
+
}
|
|
349
|
+
|
|
277
350
|
const SETUP_WORDS = /\b(?:is|are) (?:not )?(?:enabled|configured|set)\b|\bset the plugin option\b|\bsign in\b|\bnot configured\b/i
|
|
278
351
|
|
|
279
352
|
/**
|
|
@@ -284,11 +357,20 @@ const SETUP_WORDS = /\b(?:is|are) (?:not )?(?:enabled|configured|set)\b|\bset th
|
|
|
284
357
|
*/
|
|
285
358
|
export function setupNote(answer, said = "") {
|
|
286
359
|
const own = typeof answer?.setup === "string" ? answer.setup.trim() : ""
|
|
287
|
-
if (own) return
|
|
288
|
-
if (said && Array.isArray(answer?.events) && answer.events.length === 0 && SETUP_WORDS.test(said)) return
|
|
360
|
+
if (own) return redactedSetup(own)
|
|
361
|
+
if (said && Array.isArray(answer?.events) && answer.events.length === 0 && SETUP_WORDS.test(said)) return redactedSetup(said.split(/(?<=\.)\s/)[0])
|
|
289
362
|
return null
|
|
290
363
|
}
|
|
291
364
|
|
|
365
|
+
/**
|
|
366
|
+
* A source's own setup sentence, redacted: it is third-party text, and it is
|
|
367
|
+
* kept in the run's result, in the status file and in every answer that shows
|
|
368
|
+
* the run (0.10.4's gate; review F2).
|
|
369
|
+
*/
|
|
370
|
+
function redactedSetup(text) {
|
|
371
|
+
return oneLine(errorMessage(text), 400)
|
|
372
|
+
}
|
|
373
|
+
|
|
292
374
|
/** The input of the owner-only call a bind request makes (§4.5): `limit: 0` where the plugin keeps it, else `limit: 1`. */
|
|
293
375
|
export function probeInput(entry, account) {
|
|
294
376
|
return { limit: entry.ownerProbe === "limit0" ? 0 : 1, ...accountInput(entry, account) }
|
|
@@ -328,24 +410,385 @@ export function requestedSources(requests, { only, now = Date.now() } = {}) {
|
|
|
328
410
|
*
|
|
329
411
|
* @returns {null | ((source: string, input: object, sessionID?: string) => Promise<ReturnType<typeof readEventsAnswer>>)}
|
|
330
412
|
*/
|
|
331
|
-
export function hostSourceLister(ctx) {
|
|
413
|
+
export function hostSourceLister(ctx, { broker = null, brokerDetail = null, brokerState = null, timeoutMs = SOURCE_READ_TIMEOUT_MS } = {}) {
|
|
332
414
|
if (typeof ctx?.rpc !== "function") return null
|
|
333
|
-
|
|
415
|
+
const lister = async (source, input, sessionID, confirmedDigest = null) => {
|
|
334
416
|
const entry = sourceEntry(source)
|
|
335
417
|
if (!entry) throw new Error(`${oneLine(source, 40)} is not a source Memory knows`)
|
|
418
|
+
// A declared source is read through the contract's own `read`: the grant's
|
|
419
|
+
// rules, the descriptor's hash, the params and the page checks are the
|
|
420
|
+
// package's, and Memory keeps only its cursor.
|
|
421
|
+
if (entry.contract === "sources") {
|
|
422
|
+
return readSource({ entry, ctx, sessionID, account: input?.account ?? DEFAULT_ACCOUNT, scope: input?.scope, cursor: input?.cursor, limit: input?.limit, broker, brokerDetail, confirmedDigest, timeoutMs, readState: brokerState })
|
|
423
|
+
}
|
|
336
424
|
const client = ctx.rpc(rpcDefinition(entry))
|
|
337
425
|
if (typeof client?.[entry.tool] !== "function") throw new Error(`the host has no RPC method ${entry.rpc}/${entry.tool}`)
|
|
338
|
-
return readEventsAnswer(await client[entry.tool]({ sessionID, input: { ...input, ...entry.input } }), entry)
|
|
426
|
+
return readEventsAnswer(await client[entry.tool]({ sessionID, input: { ...input, ...entry.input } }), entry, input?.account ?? null)
|
|
427
|
+
}
|
|
428
|
+
// One broker deadline per tick: the collector resets it at the start of every run.
|
|
429
|
+
if (brokerState) {
|
|
430
|
+
lister.resetBrokerTick = () => {
|
|
431
|
+
brokerState.timedOut = false
|
|
432
|
+
brokerState.embedded?.clear?.()
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
return lister
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/** The host's own core RPC: jelly's `jelly.core`, which parks a plugin's ask and the host answers. */
|
|
439
|
+
export const CORE_RPC_ID = "jelly.core"
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* How long one source read gets. **Nothing bounded an embedded read:** a declared source's page was awaited
|
|
443
|
+
* through the host's tools RPC with no deadline at all (`readSource`'s `host.call`), so a hung plugin hung the
|
|
444
|
+
* collector. This is the budget proposed for both paths — a broker read and an embedded read of the same source
|
|
445
|
+
* — so a live broker that is merely slow (a 20 s provider read) is not mistaken for a silent one. 30 s is
|
|
446
|
+
* fifteen times the records client's own read timeout (`config.timeoutMs`, 2,000 ms) and well inside a
|
|
447
|
+
* 30-minute cadence.
|
|
448
|
+
*/
|
|
449
|
+
export const SOURCE_READ_TIMEOUT_MS = 30_000
|
|
450
|
+
/** The broker's default deadline: the same budget every embedded read of that source now gets. */
|
|
451
|
+
export const BROKER_TIMEOUT_MS = SOURCE_READ_TIMEOUT_MS
|
|
452
|
+
/** The deadline's own value, distinct from any answer a broker could give. */
|
|
453
|
+
const TIMEOUT = Symbol("broker timeout")
|
|
454
|
+
|
|
455
|
+
/**
|
|
456
|
+
* A transport to the host's source broker (`jelly.core.request`, kind `broker`),
|
|
457
|
+
* or null when the host gives plugins no tools RPC channel.
|
|
458
|
+
*
|
|
459
|
+
* **Never a bypass (B1).** The contract's `sourceReader` reads embedded when a
|
|
460
|
+
* transport throws or answers something that is not a broker answer, and that
|
|
461
|
+
* would read around jelly's grant and declaration checks. So this transport
|
|
462
|
+
* never throws and never answers nothing: an unreachable core plugin, a rejected
|
|
463
|
+
* call, an empty or malformed answer all become a refusal (`offered: true,
|
|
464
|
+
* ok: false, UNAVAILABLE`), which holds the binding. Only jelly's own explicit
|
|
465
|
+
* `{ offered: false }` (its broker is off) lets the read run embedded.
|
|
466
|
+
*
|
|
467
|
+
* **The capability is the resource (B3).** The broker is offered only where the
|
|
468
|
+
* host offers the `memory` resource (resource-token contract §1/§6), checked per
|
|
469
|
+
* read; every other host reads embedded and its RPC is never asked. A host that
|
|
470
|
+
* does offer it keeps the strict rule: only its explicit `{ offered: false }`
|
|
471
|
+
* reads embedded, and every failure — `rpc.unavailable`, `rpc.method_not_found`
|
|
472
|
+
* and any other — fails closed. No host identity is read.
|
|
473
|
+
* The package's `broker-client` still says never to *find* a broker by asking
|
|
474
|
+
* an RPC id: an RPC id has no owner, so another plugin could answer in the
|
|
475
|
+
* broker's name. The id is the source-contract's own.
|
|
476
|
+
*
|
|
477
|
+
* **A broker that never answers cannot stall collection (round 6).** The call is
|
|
478
|
+
* bounded by `timeoutMs` (default `SOURCE_READ_TIMEOUT_MS`, the same 30 s an
|
|
479
|
+
* embedded read gets, or the caller's), aborted when the client honours
|
|
480
|
+
* a signal, and turned into a refusal (`offered: true, ok: false, UNAVAILABLE`):
|
|
481
|
+
* the binding is held and never read embedded. Older Jelly betas, which answer
|
|
482
|
+
* nothing while their broker is off, therefore pause one binding instead of
|
|
483
|
+
* hanging the run; the next cadence tries again.
|
|
484
|
+
*
|
|
485
|
+
* @param {any} ctx the plugin context
|
|
486
|
+
* @param {{ detail?: object | null, timeoutMs?: number }} [host] the caller's deadline and detail
|
|
487
|
+
* @returns {null | ((request: object, signal?: AbortSignal) => Promise<any>)}
|
|
488
|
+
*/
|
|
489
|
+
export function hostBrokerTransport(ctx, { detail = null, timeoutMs = SOURCE_READ_TIMEOUT_MS, state = null } = {}) {
|
|
490
|
+
if (typeof ctx?.rpc !== "function") return null
|
|
491
|
+
const refuse = (message) => ({ offered: true, ok: false, code: "UNAVAILABLE", message, next: "Retry on the next cadence; nothing was read." })
|
|
492
|
+
return async (request, signal = undefined) => {
|
|
493
|
+
// The interim contract fact, asked per read: only a host that offers the
|
|
494
|
+
// `memory` resource can have a broker. Plain OpenCode hands a callable
|
|
495
|
+
// `rpc` too, and answers an unknown id with `rpc.unavailable`
|
|
496
|
+
// (`packages/core/src/rpc.ts:108-114`), so its channel must never be asked.
|
|
497
|
+
// A host that does offer the resource keeps the rule below: only its own
|
|
498
|
+
// `{ offered: false }` lets a read run embedded; every failure fails closed.
|
|
499
|
+
if (!memoryResourceAvailable()) return { offered: false }
|
|
500
|
+
// One deadline per tick (N7b): after the first broker timeout this run refuses the rest at once, so a
|
|
501
|
+
// host whose broker never answers costs one deadline, not one per binding. The next tick tries again.
|
|
502
|
+
if (state && state.timedOut === true) {
|
|
503
|
+
return refuse("jelly's broker did not answer earlier in this run, so nothing was read; the binding is retried on its next cadence")
|
|
504
|
+
}
|
|
505
|
+
let settled
|
|
506
|
+
const controller = new AbortController()
|
|
507
|
+
const onAbort = () => controller.abort()
|
|
508
|
+
signal?.addEventListener?.("abort", onAbort, { once: true })
|
|
509
|
+
let timer
|
|
510
|
+
const deadline = new Promise((resolve) => {
|
|
511
|
+
timer = setTimeout(() => (controller.abort(TIMEOUT), resolve(TIMEOUT)), timeoutMs)
|
|
512
|
+
})
|
|
513
|
+
try {
|
|
514
|
+
const client = ctx.rpc({ id: CORE_RPC_ID, events: {}, methods: { request: { input: { type: "object" }, output: { type: "object" } } } })
|
|
515
|
+
if (typeof client?.request !== "function") return refuse("jelly's core plugin is not answering, so its broker cannot be asked")
|
|
516
|
+
// The core plugin settles the parked call with the host's answer (`{answer, seal}`); a client that honours
|
|
517
|
+
// the signal aborts the parked request, so no call is left dangling.
|
|
518
|
+
// OpenCode's Promise adapter reads `options.signal` (`@opencode/plugin` 2.2.6
|
|
519
|
+
// `dist/promise/adapter.js:52-57`: `(input, options) => run(method(input), { signal: options?.signal })`),
|
|
520
|
+
// so the abort must travel as `{ signal }`; a bare signal is ignored and the call stays parked.
|
|
521
|
+
const call = client.request({ id: `mem-broker-${createHash("sha256").update(JSON.stringify(request)).digest("hex").slice(0, 16)}-${Date.now()}`, kind: "broker", body: request, keep: false }, { signal: controller.signal })
|
|
522
|
+
const raced = await Promise.race([call, deadline])
|
|
523
|
+
if (raced === TIMEOUT) {
|
|
524
|
+
if (state) state.timedOut = true
|
|
525
|
+
return refuse("jelly's broker did not answer in time, so nothing was read; the binding is retried on its next cadence")
|
|
526
|
+
}
|
|
527
|
+
settled = raced
|
|
528
|
+
} catch (error) {
|
|
529
|
+
if (controller.signal.aborted && controller.signal.reason === TIMEOUT) {
|
|
530
|
+
if (state) state.timedOut = true
|
|
531
|
+
return refuse("jelly's broker did not answer in time, so nothing was read; the binding is retried on its next cadence")
|
|
532
|
+
}
|
|
533
|
+
// A crashed or restarted core plugin, a missing method, or any other
|
|
534
|
+
// host error: a refusal, never an embedded read. On a host that offers
|
|
535
|
+
// `memory`, only its explicit `{ offered: false }` reads embedded.
|
|
536
|
+
return refuse(`jelly's broker could not be asked: ${oneLine(errorMessage(error), 200)}`)
|
|
537
|
+
} finally {
|
|
538
|
+
clearTimeout(timer)
|
|
539
|
+
signal?.removeEventListener?.("abort", onAbort)
|
|
540
|
+
}
|
|
541
|
+
const answer = settled?.answer
|
|
542
|
+
if (!isBrokerAnswer(answer)) return refuse(`jelly's broker answered nothing the contract defines`)
|
|
543
|
+
// The original code, before the package aliases it (NOT_GRANTED is a withdrawn jelly grant, not a
|
|
544
|
+
// changed declaration). The classifier uses this, never the message.
|
|
545
|
+
if (detail && answer.offered === true && answer.ok === false && typeof answer.code === "string") detail.code = answer.code
|
|
546
|
+
return answer
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* One page of a declared source, read through the contract's `read`.
|
|
552
|
+
*
|
|
553
|
+
* The host it is given is the binding's own session (its rules were made for
|
|
554
|
+
* this source and account), and the call is the plugin's tools-RPC method. The
|
|
555
|
+
* grant is Memory's, for the account and the scope the person confirmed, and
|
|
556
|
+
* for the descriptor hash of the install on disk: a changed descriptor is
|
|
557
|
+
* `PERMISSION_DENIED`, never a silent read.
|
|
558
|
+
*
|
|
559
|
+
* @param {{ entry: object, ctx: any, sessionID: string, account: string, scope?: object, cursor?: string, limit?: number, now?: number }} input
|
|
560
|
+
* @returns {Promise<object>} the page, in the shape the collector stores
|
|
561
|
+
*/
|
|
562
|
+
export async function readSource({ entry, ctx, sessionID, account, scope, cursor, limit = PAGE_LIMIT, now = Date.now(), broker = null, brokerDetail = null, confirmedDigest = null, timeoutMs = SOURCE_READ_TIMEOUT_MS, readState = null }) {
|
|
563
|
+
if (entry?.contract !== "sources" || !entry.item) throw new Error(`${oneLine(String(entry?.source ?? "?"), 40)} is not a declared source`)
|
|
564
|
+
if (typeof ctx?.rpc !== "function") throw new Error("this host gives plugins no tools RPC, so a source cannot be read")
|
|
565
|
+
const methods = { [entry.tool]: { input: { type: "object" }, output: { type: "object" } } }
|
|
566
|
+
const host = {
|
|
567
|
+
async openSession() {
|
|
568
|
+
return { sessionID, async close() {} }
|
|
569
|
+
},
|
|
570
|
+
async call({ method, input }) {
|
|
571
|
+
const client = ctx.rpc({ id: entry.rpc, events: {}, methods })
|
|
572
|
+
if (typeof client?.[method] !== "function") {
|
|
573
|
+
// The one refusal D11 needs: the contract's probe method, which every
|
|
574
|
+
// declared source implements. Declaring `sources.probe.action` beside it
|
|
575
|
+
// is optional. No events are read, here or anywhere.
|
|
576
|
+
const error = new Error(
|
|
577
|
+
`${oneLine(String(entry.source), 40)} implements no ${oneLine(String(method), 20)} method: the contract's probe is how Memory reads who it is signed in as (D11). Declaring sources.probe.action is optional; implementing the method is not. Nothing was read. Next step: add the ${oneLine(String(method), 20)} method to ${oneLine(String(entry.package ?? "its plugin"), 40)}'s tools RPC, then bind the source again.`,
|
|
578
|
+
)
|
|
579
|
+
error.code = "SCHEMA_VIOLATION"
|
|
580
|
+
throw error
|
|
581
|
+
}
|
|
582
|
+
// The embedded read’s budget, the same one the broker gets, and per plugin per tick: once one plugin’s
|
|
583
|
+
// read times out, its other bindings this run are held at once, and another plugin is unaffected (N8b).
|
|
584
|
+
const embedded = readState ? (readState.embedded ??= new Set()) : null
|
|
585
|
+
if (embedded?.has(entry.package)) {
|
|
586
|
+
const error = new Error("UNAVAILABLE: this source's plugin did not answer earlier in this run, so nothing was read. Next step: retry on the next cadence.")
|
|
587
|
+
error.code = "UNAVAILABLE"
|
|
588
|
+
throw error
|
|
589
|
+
}
|
|
590
|
+
try {
|
|
591
|
+
return await boundedSourceCall(client, method, input, sessionID, timeoutMs)
|
|
592
|
+
} catch (error) {
|
|
593
|
+
if (error?.timedOut === true) embedded?.add(entry.package)
|
|
594
|
+
throw error
|
|
595
|
+
}
|
|
596
|
+
},
|
|
597
|
+
}
|
|
598
|
+
const confirmed = isPlainObject(scope) && Object.keys(scope).length ? scope : undefined
|
|
599
|
+
// The digest the **person confirmed** (the consent's), never merely the one installed now: jelly's broker
|
|
600
|
+
// compares it with its own installed declaration and refuses a read under another one (B4).
|
|
601
|
+
const digest = typeof confirmedDigest === "string" && /^sha256:[0-9a-f]{64}$/.test(confirmedDigest) ? confirmedDigest : entry.descriptorHash
|
|
602
|
+
const installed = { package: entry.package, version: entry.version ?? "0.0.0", directory: "", sources: entry.descriptor, hash: digest }
|
|
603
|
+
const grant = { consumer: "memory", source: entry.source, account: account ?? DEFAULT_ACCOUNT, ...(confirmed ? { scope: confirmed } : {}), descriptorHash: digest }
|
|
604
|
+
// The package's reader: jelly's host broker first when it is offered (and the host gave one), else the
|
|
605
|
+
// embedded read. Its request carries `package`, `version`, `descriptorHash` and `consumer: "memory"`, so a
|
|
606
|
+
// broker whose installed declaration differs refuses instead of serving a read the person did not confirm.
|
|
607
|
+
const outcome = await sourceReader({ host, ...(broker ? { broker } : {}) }).read({
|
|
608
|
+
installed,
|
|
609
|
+
source: entry.source,
|
|
610
|
+
grant,
|
|
611
|
+
consumer: "memory",
|
|
612
|
+
...(confirmed ? { scope: confirmed } : {}),
|
|
613
|
+
...(cursor ? { cursor } : {}),
|
|
614
|
+
limit,
|
|
615
|
+
context: { now, timeZone: "UTC" },
|
|
616
|
+
})
|
|
617
|
+
if (!outcome.ok) {
|
|
618
|
+
const error = new Error(outcome.failure.message)
|
|
619
|
+
error.code = outcome.failure.code
|
|
620
|
+
error.next = outcome.failure.next
|
|
621
|
+
// The classifier's facts: how the read was made, and the refusal's own code before aliasing.
|
|
622
|
+
error.broker = outcome.via === "broker"
|
|
623
|
+
error.brokerCode = brokerDetail?.code ?? null
|
|
624
|
+
throw error
|
|
625
|
+
}
|
|
626
|
+
const page = outcome.page
|
|
627
|
+
return {
|
|
628
|
+
source: entry.source,
|
|
629
|
+
pageSource: page.source,
|
|
630
|
+
account: page.account,
|
|
631
|
+
owner: page.owner ?? null,
|
|
632
|
+
events: page.events,
|
|
633
|
+
cursor: page.cursor ?? "",
|
|
634
|
+
more: page.more === true,
|
|
635
|
+
refused: Array.isArray(page.refused) ? page.refused : [],
|
|
636
|
+
failed: Array.isArray(page.failed) ? page.failed : [],
|
|
637
|
+
// The page's own echo of what it read: kept, so the store can refuse a page
|
|
638
|
+
// that read wider than the person confirmed (plugin_scope).
|
|
639
|
+
scope: isPlainObject(page.scope) ? page.scope : null,
|
|
640
|
+
gap: isPlainObject(page.gap) ? page.gap : null,
|
|
641
|
+
cost: isPlainObject(page.cost) ? page.cost : null,
|
|
642
|
+
setup: null,
|
|
339
643
|
}
|
|
340
644
|
}
|
|
341
645
|
|
|
646
|
+
/**
|
|
647
|
+
* One call of a plugin's tools RPC, inside the source's read budget, aborted in the shape OpenCode reads
|
|
648
|
+
* (`@opencode/plugin` 2.2.6 `dist/promise/adapter.js:52-57`: `(input, options) => run(method(input), { signal: options?.signal })`).
|
|
649
|
+
* A call that outlives the budget is `UNAVAILABLE`, and the binding holds.
|
|
650
|
+
*/
|
|
651
|
+
async function boundedSourceCall(client, method, input, sessionID, timeoutMs) {
|
|
652
|
+
const controller = new AbortController()
|
|
653
|
+
// "Timed out" is decided by OUR deadline, not by how the promise settled: a plugin that rejects the moment it
|
|
654
|
+
// is aborted (which OpenCode's adapter does) rejects before the deadline's own resolution is delivered, and
|
|
655
|
+
// that must still count for the per-plugin circuit.
|
|
656
|
+
let fired = false
|
|
657
|
+
let timer
|
|
658
|
+
const deadline = new Promise((resolve) => {
|
|
659
|
+
timer = setTimeout(() => {
|
|
660
|
+
fired = true
|
|
661
|
+
controller.abort("source read timeout")
|
|
662
|
+
resolve(TIMEOUT)
|
|
663
|
+
}, timeoutMs)
|
|
664
|
+
})
|
|
665
|
+
try {
|
|
666
|
+
const raced = await Promise.race([client[method]({ sessionID, input }, { signal: controller.signal }), deadline])
|
|
667
|
+
if (raced === TIMEOUT || fired) {
|
|
668
|
+
const error = new Error("UNAVAILABLE: the source did not answer within its read budget, so nothing was read. Next step: retry on the next cadence.")
|
|
669
|
+
error.code = "UNAVAILABLE"
|
|
670
|
+
error.timedOut = true
|
|
671
|
+
throw error
|
|
672
|
+
}
|
|
673
|
+
return raced
|
|
674
|
+
} catch (error) {
|
|
675
|
+
if (fired && (error?.code === undefined || error?.code === "UNAVAILABLE")) {
|
|
676
|
+
const timeout = new Error("UNAVAILABLE: the source did not answer within its read budget, so nothing was read. Next step: retry on the next cadence.")
|
|
677
|
+
timeout.code = "UNAVAILABLE"
|
|
678
|
+
timeout.timedOut = true
|
|
679
|
+
throw timeout
|
|
680
|
+
}
|
|
681
|
+
throw error
|
|
682
|
+
} finally {
|
|
683
|
+
clearTimeout(timer)
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* One `probe` of a declared source, through the contract (D11): the source's own
|
|
689
|
+
* probe method, in the binding's session, answers the signed-in account and its
|
|
690
|
+
* owner. It reads no content, and it replaces the owner-only
|
|
691
|
+
* `events_list({ limit: 0 })` page for a source that has moved to the contract.
|
|
692
|
+
*
|
|
693
|
+
* The host is the binding's own session, as a page read's is: its rules allow
|
|
694
|
+
* the source's events action, its operations and its probe action.
|
|
695
|
+
*
|
|
696
|
+
* @param {{ entry: object, ctx: any, sessionID: string, account: string }} input
|
|
697
|
+
* @returns {Promise<{ account: string, owner: object, scopes?: string[] }>}
|
|
698
|
+
*/
|
|
699
|
+
export async function probeSource({ entry, ctx, sessionID, account, timeoutMs = SOURCE_READ_TIMEOUT_MS }) {
|
|
700
|
+
if (entry?.contract !== "sources" || !entry.item) throw new Error(`${oneLine(String(entry?.source ?? "?"), 40)} is not a declared source`)
|
|
701
|
+
// D11 reads the owner identity through the contract's `probe` and nothing
|
|
702
|
+
// else; the contract refuses `events_list({ limit: 0 })` for a declared
|
|
703
|
+
// source, so there is no page to fall back to (contract-v010 review, F5).
|
|
704
|
+
// `sources.probe` is **optional** in contract v0.1.0: its `probeRules` fences
|
|
705
|
+
// the probe action with `probeAction ?? methodAction(source)`, and this build's
|
|
706
|
+
// `sourceAllowRules` falls back the same way, so a plugin that implements the
|
|
707
|
+
// probe *method* without declaring the action is conformant and is probed. What
|
|
708
|
+
// is refused is a source with no probe method at all (the check in `call`).
|
|
709
|
+
if (typeof ctx?.rpc !== "function") throw new Error("this host gives plugins no tools RPC, so a source cannot be probed")
|
|
710
|
+
const methods = { probe: { input: { type: "object" }, output: { type: "object" } } }
|
|
711
|
+
/** The one refusal D11 needs: the contract's probe method, which every declared source implements. */
|
|
712
|
+
const missingMethod = (method) => {
|
|
713
|
+
// Declaring `sources.probe.action` beside the method is optional. No events
|
|
714
|
+
// are read, here or anywhere.
|
|
715
|
+
const error = new Error(
|
|
716
|
+
`${oneLine(String(entry.source), 40)} implements no ${oneLine(String(method), 20)} method: the contract's probe is how Memory reads who it is signed in as (D11). Declaring sources.probe.action is optional; implementing the method is not. Nothing was read. Next step: add the ${oneLine(String(method), 20)} method to ${oneLine(String(entry.package ?? "its plugin"), 40)}'s tools RPC, then bind the source again.`,
|
|
717
|
+
)
|
|
718
|
+
error.code = "SCHEMA_VIOLATION"
|
|
719
|
+
return error
|
|
720
|
+
}
|
|
721
|
+
const host = {
|
|
722
|
+
async openSession() {
|
|
723
|
+
return { sessionID, async close() {} }
|
|
724
|
+
},
|
|
725
|
+
async call({ method, input }) {
|
|
726
|
+
const client = ctx.rpc({ id: entry.rpc, events: {}, methods })
|
|
727
|
+
// Memory's own definition always carries the method, so a real host has it:
|
|
728
|
+
// the engine's own type is what a plugin without it answers (below).
|
|
729
|
+
if (typeof client?.[method] !== "function") throw missingMethod(method)
|
|
730
|
+
try {
|
|
731
|
+
return await boundedSourceCall(client, method, input, sessionID, timeoutMs)
|
|
732
|
+
} catch (error) {
|
|
733
|
+
if (error?.type === "rpc.method_not_found") throw missingMethod(method)
|
|
734
|
+
throw error
|
|
735
|
+
}
|
|
736
|
+
},
|
|
737
|
+
}
|
|
738
|
+
const installed = { package: entry.package, version: entry.version ?? "0.0.0", directory: "", sources: entry.descriptor, hash: entry.descriptorHash }
|
|
739
|
+
const outcome = await probe(host, { installed, source: entry.source, account: account ?? DEFAULT_ACCOUNT, identityOperations: ["probe"] })
|
|
740
|
+
if (!outcome.ok) {
|
|
741
|
+
const error = new Error(outcome.failure.message)
|
|
742
|
+
error.code = outcome.failure.code
|
|
743
|
+
error.next = outcome.failure.next
|
|
744
|
+
throw error
|
|
745
|
+
}
|
|
746
|
+
// The contract's one probe check, called by the consumer too: the answer names the source and the account,
|
|
747
|
+
// carries owner { id, label } (never null) and trust: "untrusted", and reads no content (D8, D230).
|
|
748
|
+
const problems = probeResultProblems(outcome.answer, { ...(entry.pageSource ? { source: entry.pageSource } : {}), account: account ?? DEFAULT_ACCOUNT })
|
|
749
|
+
if (problems.length > 0) {
|
|
750
|
+
const error = new Error(`SCHEMA_VIOLATION: ${entry.source}'s probe answer is not the contract's: ${problems[0]} Next step: report it to the plugin's maintainers.`)
|
|
751
|
+
error.code = "SCHEMA_VIOLATION"
|
|
752
|
+
throw error
|
|
753
|
+
}
|
|
754
|
+
return outcome.answer
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
/** A declared source's probe through the host's tools RPC; null when the host has no RPC client. */
|
|
758
|
+
export function hostSourceProber(ctx) {
|
|
759
|
+
if (typeof ctx?.rpc !== "function") return null
|
|
760
|
+
return async (source, input, sessionID) => {
|
|
761
|
+
const entry = sourceEntry(source)
|
|
762
|
+
if (!entry) throw new Error(`${oneLine(source, 40)} is not a source Memory knows`)
|
|
763
|
+
if (entry.contract !== "sources") throw new Error(`${oneLine(source, 40)} is read through its events_list, not probed`)
|
|
764
|
+
return probeSource({ entry, ctx, sessionID, account: input?.account ?? DEFAULT_ACCOUNT })
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
/**
|
|
769
|
+
* One refused event, kept in the run's own words (DESIGN-jelly-sources §2.3.1),
|
|
770
|
+
* because a count is not an account: the contract's policy advances the cursor
|
|
771
|
+
* past `refused[]` (`cursorAdvance`), so the id and the reason travel with the
|
|
772
|
+
* result to `mem_spaces` health and are never only a number.
|
|
773
|
+
*/
|
|
774
|
+
function noteRefusal(result, refusal) {
|
|
775
|
+
if (!isPlainObject(refusal)) return
|
|
776
|
+
const reason = refusal.reason ?? refusal.detail
|
|
777
|
+
// The reason is the source's own words, and it is kept in the result, in the
|
|
778
|
+
// status file and in health: redacted where it is kept, never only where it
|
|
779
|
+
// is shown (0.10.4's gate; review F2).
|
|
780
|
+
result.refused_reason = oneLine(errorMessage(typeof reason === "string" && reason ? reason : "the source refused an event"), 200)
|
|
781
|
+
const id = refusal.external_id === undefined || refusal.external_id === null ? null : oneLine(String(refusal.external_id), 100)
|
|
782
|
+
result.refused_id = id
|
|
783
|
+
}
|
|
784
|
+
|
|
342
785
|
/**
|
|
343
786
|
* The collector's core: schedule upkeep, one collection, and the owners a
|
|
344
787
|
* refused ingest reported.
|
|
345
788
|
*
|
|
346
|
-
* @param {{ scheduler?: { list(): Promise<any[]>, create(input: any): Promise<any>, remove(id: string): Promise<void> } | null, lister?: ReturnType<typeof hostSourceLister>, log?: any }} [input]
|
|
789
|
+
* @param {{ scheduler?: { list(): Promise<any[]>, create(input: any): Promise<any>, remove(id: string): Promise<void> } | null, lister?: ReturnType<typeof hostSourceLister>, prober?: ReturnType<typeof hostSourceProber>, log?: any }} [input]
|
|
347
790
|
*/
|
|
348
|
-
export function createCollector({ scheduler = null, lister = null, log } = {}) {
|
|
791
|
+
export function createCollector({ scheduler = null, lister = null, prober = null, log } = {}) {
|
|
349
792
|
const reported = new Map()
|
|
350
793
|
|
|
351
794
|
function noteOwner(source, account, owner) {
|
|
@@ -409,7 +852,9 @@ export function createCollector({ scheduler = null, lister = null, log } = {}) {
|
|
|
409
852
|
* due?: (binding: { source: string, account: string }) => boolean,
|
|
410
853
|
* }} input
|
|
411
854
|
*/
|
|
412
|
-
async function runOnce({ sources, only, cursor, ingest, sessionID, results = [], due = () => true }) {
|
|
855
|
+
async function runOnce({ sources, only, cursor, ingest, sessionID, results = [], due = () => true, pause = null }) {
|
|
856
|
+
// One broker deadline per tick (N7b): a broker that timed out this run does not cost each later binding.
|
|
857
|
+
lister?.resetBrokerTick?.()
|
|
413
858
|
for (const binding of collectableSources(sources, { only })) {
|
|
414
859
|
if (!due(binding)) {
|
|
415
860
|
results.push({ ...binding, pages: 0, accepted: 0, duplicates: 0, refused: 0, more: false, notDue: true, everyMinutes: cadenceMs(binding.source) / 60_000, done: true })
|
|
@@ -429,29 +874,162 @@ export function createCollector({ scheduler = null, lister = null, log } = {}) {
|
|
|
429
874
|
continue
|
|
430
875
|
}
|
|
431
876
|
let at = typeof state.cursor === "string" && state.cursor ? state.cursor : undefined
|
|
877
|
+
const scope = isPlainObject(state.scope) ? state.scope : {}
|
|
878
|
+
// The owner the person confirmed: a page that answers another is a rebind
|
|
879
|
+
// (§15.3), paused here with both labels — never read under the new one.
|
|
880
|
+
const bound = isPlainObject(state.owner) && typeof state.owner.id === "string" && state.owner.id ? state.owner : null
|
|
881
|
+
// The catalog entry this binding reads under: a declared source's page is
|
|
882
|
+
// ingested under the binding's own account (G2), a legacy one as before.
|
|
883
|
+
const boundEntry = sourceEntry(binding.source)
|
|
884
|
+
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
885
|
+
try {
|
|
432
886
|
for (let page = 0; page < PAGES_PER_SOURCE; page += 1) {
|
|
433
|
-
const answer = await lister(binding.source, listInput(binding, at), session)
|
|
887
|
+
const answer = await lister(binding.source, listInput(binding, at, scope), session, isPlainObject(state) ? state.digest ?? null : null)
|
|
434
888
|
if (!answer || !answer.cursor) {
|
|
435
889
|
result.error = "the source's events_list gave no events and cursor"
|
|
436
890
|
break
|
|
437
891
|
}
|
|
438
|
-
|
|
892
|
+
if (bound && isPlainObject(answer.owner) && typeof answer.owner.id === "string" && answer.owner.id && answer.owner.id !== bound.id) {
|
|
893
|
+
const was = bound.label ?? bound.id
|
|
894
|
+
const now = answer.owner.label ?? answer.owner.id
|
|
895
|
+
const reason = `${binding.source} (${binding.account}) was bound to ${was}; the provider now answers ${now}. Next step: rebind it in Memory's console to confirm the new account.`
|
|
896
|
+
result.code = "OWNER_CHANGED"
|
|
897
|
+
result.action = "pause"
|
|
898
|
+
result.error = reason
|
|
899
|
+
if (typeof pause === "function") {
|
|
900
|
+
try {
|
|
901
|
+
await pause(binding, reason, "OWNER_CHANGED")
|
|
902
|
+
result.paused = true
|
|
903
|
+
} catch (error) {
|
|
904
|
+
result.pause_error = oneLine(errorMessage(error), 200)
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
break
|
|
908
|
+
}
|
|
909
|
+
// The answer's own account, and the binding's own source (§4.5): a
|
|
910
|
+
// declared source's page names the plugin's name for it (`slack`), while
|
|
911
|
+
// the person confirmed the descriptor's id (`slack.workspace`), and the
|
|
912
|
+
// store keys the consent and the cursor by that id — as a job does.
|
|
439
913
|
if (answer.setup) result.setup = answer.setup
|
|
440
|
-
|
|
914
|
+
// The contract's page rules (§8.3.4): `failed[]` holds the cursor back,
|
|
915
|
+
// `refused[]` does not, and a `gap` advances with the hole recorded.
|
|
916
|
+
const move = cursorAdvance(answer)
|
|
917
|
+
if (Array.isArray(answer.refused) && answer.refused.length) {
|
|
918
|
+
result.refused += answer.refused.length
|
|
919
|
+
noteRefusal(result, answer.refused.at(-1))
|
|
920
|
+
}
|
|
921
|
+
if (Array.isArray(answer.failed) && answer.failed.length) {
|
|
922
|
+
result.failed = answer.failed.length
|
|
923
|
+
// `held` is the page's code, or its own words when it sent none: it
|
|
924
|
+
// is kept in the run's result and the status file, so redacted where
|
|
925
|
+
// it is kept (0.11 review S1).
|
|
926
|
+
result.held = move.hidden === null || move.hidden === undefined ? move.hidden : oneLine(errorMessage(move.hidden), 100)
|
|
927
|
+
}
|
|
928
|
+
if (answer.gap) result.gap = answer.gap
|
|
929
|
+
// The page's own echo of what it read, when it gives one: the store
|
|
930
|
+
// refuses a page outside the person's confirmation (plugin_scope). A
|
|
931
|
+
// page that echoes nothing is sent the consented scope, which the read
|
|
932
|
+
// carried, so a scoped binding is not refused for saying nothing.
|
|
933
|
+
const echo = isPlainObject(answer.scope) ? answer.scope : null
|
|
934
|
+
// **The binding's account, for a declared source** (G2). A source with
|
|
935
|
+
// `accounts.named: false` reads one account, and the person's binding is
|
|
936
|
+
// keyed by the account its probe reported (`history.sessions` answers
|
|
937
|
+
// the machine it indexed, `linear.activity` a workspace user). The
|
|
938
|
+
// page's own account is the same one; sending it for a legacy source
|
|
939
|
+
// keeps that path as it was.
|
|
940
|
+
const saved = await ingest({
|
|
941
|
+
source: binding.source,
|
|
942
|
+
account: boundEntry?.contract === "sources" ? binding.account : answer.account,
|
|
943
|
+
owner: answer.owner,
|
|
944
|
+
events: answer.events,
|
|
945
|
+
...(echo && Object.keys(echo).length ? { scope: echo } : Object.keys(scope).length ? { scope } : {}),
|
|
946
|
+
// Held: the events are stored idempotently and the cursor does not move.
|
|
947
|
+
...(move.advance && answer.cursor ? { cursor: answer.cursor } : {}),
|
|
948
|
+
...(answer.gap ? { gap: answer.gap } : {}),
|
|
949
|
+
...(answer.cost ? { cost: answer.cost } : {}),
|
|
950
|
+
})
|
|
441
951
|
result.pages += 1
|
|
442
952
|
result.accepted += Number(saved?.accepted) || 0
|
|
443
953
|
result.duplicates += Number(saved?.duplicates) || 0
|
|
444
|
-
|
|
445
|
-
|
|
954
|
+
// The page's own `refused[]` and the store's are disjoint — the store
|
|
955
|
+
// is sent only the page's events — so both are added: every refused
|
|
956
|
+
// event is counted once (DESIGN-jelly-sources §2.3.1).
|
|
957
|
+
if (saved?.refused !== undefined) result.refused += Number(saved.refused) || 0
|
|
958
|
+
if (saved?.newest_refusal) noteRefusal(result, saved.newest_refusal)
|
|
959
|
+
if (saved?.resync === false) {
|
|
960
|
+
result.error = "the source restarted its window twice today; the binding waits until tomorrow rather than read again blind"
|
|
961
|
+
result.action = "pause"
|
|
962
|
+
break
|
|
963
|
+
}
|
|
964
|
+
if (move.advance && saved?.cursor_saved !== true) {
|
|
446
965
|
result.error = saved?.error || "Memory did not save the cursor, so collection stopped here"
|
|
447
966
|
break
|
|
448
967
|
}
|
|
968
|
+
if (!move.advance) break
|
|
449
969
|
at = answer.cursor
|
|
450
970
|
result.more = answer.more
|
|
451
971
|
if (!answer.more) break
|
|
452
972
|
}
|
|
973
|
+
break
|
|
974
|
+
} catch (error) {
|
|
975
|
+
// §8.3.4: a code is a pause, a hold or a reset. A reset reads once more
|
|
976
|
+
// from the window; a pause pauses the binding through its callback.
|
|
977
|
+
const failure = failureFromError(error)
|
|
978
|
+
// A PERMISSION_DENIED is decided by fact, never by its message: the broker's refusal is a changed
|
|
979
|
+
// declaration only when the digest installed now differs from the one the person confirmed. Equal
|
|
980
|
+
// digests mean the access itself was refused (a withdrawn grant, or the provider), which re-consenting
|
|
981
|
+
// cannot fix, so that state is `access_refused` with its own sentence. An embedded refusal is the
|
|
982
|
+
// source's own, and keeps `failureAction`'s meaning.
|
|
983
|
+
const entry = sourceEntry(binding.source)
|
|
984
|
+
const confirmed = isPlainObject(state) && typeof state.digest === "string" ? state.digest : null
|
|
985
|
+
const installed = typeof entry?.descriptorHash === "string" ? entry.descriptorHash : null
|
|
986
|
+
const viaBroker = error?.broker === true
|
|
987
|
+
const changed = failure.code === "PERMISSION_DENIED" && viaBroker && (!confirmed || !installed || installed !== confirmed)
|
|
988
|
+
const accessRefused = failure.code === "PERMISSION_DENIED" && viaBroker && !changed
|
|
989
|
+
const action = changed || accessRefused ? { action: "pause", reset: "none", resync: false } : failureAction(failure.code)
|
|
990
|
+
result.code = changed ? "DECLARATION_CHANGED" : accessRefused ? "ACCESS_REFUSED" : failure.code
|
|
991
|
+
const refusalMessage = "Memory's access to this source was refused. If you withdrew it, grant it again where you withdrew it."
|
|
992
|
+
result.error = changed ? "the source changed its declaration or its grant; confirm it again" : accessRefused ? refusalMessage : failure.message
|
|
993
|
+
result.action = action.action
|
|
994
|
+
if (action.action === "reset" && attempt === 1) {
|
|
995
|
+
at = undefined
|
|
996
|
+
result.reset = action.reset
|
|
997
|
+
// The read that asked for the reset is history: what this run reports
|
|
998
|
+
// as its outcome is the retry's own, not the code it recovered from.
|
|
999
|
+
result.code = undefined
|
|
1000
|
+
result.error = undefined
|
|
1001
|
+
result.action = undefined
|
|
1002
|
+
continue
|
|
1003
|
+
}
|
|
1004
|
+
if (action.action === "pause" && typeof pause === "function") {
|
|
1005
|
+
// The store holds the pause; this run only says whether it took.
|
|
1006
|
+
try {
|
|
1007
|
+
// An account that is not the bound one says so in the store: the person confirms the right account.
|
|
1008
|
+
const needsAccount = failure.code === "ACCOUNT_MISMATCH" || failure.code === "ACCOUNT_UNKNOWN"
|
|
1009
|
+
await pause(
|
|
1010
|
+
binding,
|
|
1011
|
+
changed
|
|
1012
|
+
? "the source changed its declaration or its grant; confirm it again"
|
|
1013
|
+
: accessRefused
|
|
1014
|
+
? "Memory's access to this source was refused. If you withdrew it, grant it again where you withdrew it."
|
|
1015
|
+
: needsAccount
|
|
1016
|
+
? action.reason
|
|
1017
|
+
: failure.message,
|
|
1018
|
+
changed ? "DECLARATION_CHANGED" : accessRefused ? "ACCESS_REFUSED" : failure.code,
|
|
1019
|
+
)
|
|
1020
|
+
result.paused = true
|
|
1021
|
+
} catch (error) {
|
|
1022
|
+
result.pause_error = oneLine(errorMessage(error), 200)
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
break
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
453
1028
|
} catch (error) {
|
|
454
|
-
|
|
1029
|
+
const failure = failureFromError(error)
|
|
1030
|
+
result.error = failure.message
|
|
1031
|
+
result.code = failure.code
|
|
1032
|
+
result.action = failureAction(failure.code).action
|
|
455
1033
|
} finally {
|
|
456
1034
|
result.done = true
|
|
457
1035
|
}
|
|
@@ -484,14 +1062,29 @@ export function createCollector({ scheduler = null, lister = null, log } = {}) {
|
|
|
484
1062
|
}
|
|
485
1063
|
let body
|
|
486
1064
|
try {
|
|
487
|
-
|
|
488
|
-
|
|
1065
|
+
// A declared source has the contract's `probe` (D11); the owner-only
|
|
1066
|
+
// `events_list({ limit: 0 })` page is the previous contract's.
|
|
1067
|
+
const answer = entry.contract === "sources"
|
|
1068
|
+
? await (prober ? prober(asked.source, { account: asked.account }, session) : Promise.reject(new Error("this host gives plugins no tools RPC, so a source cannot be probed")))
|
|
1069
|
+
: await (lister ? lister(asked.source, probeInput(entry, asked.account), session) : Promise.reject(new Error("this host gives plugins no way to call another plugin's events_list")))
|
|
489
1070
|
if (answer?.owner?.id) {
|
|
490
1071
|
const owner = { id: oneLine(answer.owner.id, 300), label: oneLine(answer.owner.label ?? answer.owner.email ?? answer.owner.id, 300) }
|
|
491
|
-
body = {
|
|
1072
|
+
body = {
|
|
1073
|
+
source: asked.source,
|
|
1074
|
+
account: asked.account,
|
|
1075
|
+
owner,
|
|
1076
|
+
reported_account: answer.account,
|
|
1077
|
+
// The installed declaration's digest, read from disk: a re-consent records the digest the installed
|
|
1078
|
+
// package has now, so jelly never refuses a read made under the old one.
|
|
1079
|
+
...(entry.descriptorHash ? { digest: entry.descriptorHash } : {}),
|
|
1080
|
+
// The note is the source's or the plugin's own sentence: it is
|
|
1081
|
+
// reported to the store (`plugin_requests.note`) and shown from it,
|
|
1082
|
+
// so it is redacted here, where it is written (review F1/F2).
|
|
1083
|
+
...(answer.setup || entry.setupOptions ? { note: oneLine(errorMessage(answer.setup || entry.setupOptions), 400) } : {}),
|
|
1084
|
+
}
|
|
492
1085
|
probe.owner = owner
|
|
493
1086
|
} else {
|
|
494
|
-
body = { source: asked.source, account: asked.account, note: answer?.setup || `${entry.package}'s events_list named no signed-in account; sign in to it in its own plugin.` }
|
|
1087
|
+
body = { source: asked.source, account: asked.account, note: oneLine(errorMessage(answer?.setup || `${entry.package}'s events_list named no signed-in account; sign in to it in its own plugin.`), 400) }
|
|
495
1088
|
}
|
|
496
1089
|
} catch (error) {
|
|
497
1090
|
body = { source: asked.source, account: asked.account, note: oneLine(errorMessage(error), 400) }
|