@runeditrun/rer 0.2.100 → 0.2.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -102
- package/cli/README.md +16 -89
- package/cli/app-address.mjs +2 -0
- package/cli/business/api.mjs +2 -1
- package/cli/business/choices.mjs +1 -1
- package/cli/business/command.mjs +36 -30
- package/cli/business/connect.mjs +1 -1
- package/cli/business/lease.mjs +0 -9
- package/cli/business/local-commands.mjs +1 -27
- package/cli/business/outbound.mjs +2 -14
- package/cli/business/questions.mjs +85 -48
- package/cli/business/service.mjs +1 -24
- package/cli/business/setup-command.mjs +33 -26
- package/cli/business/state.mjs +1 -1
- package/cli/business/supervisor.mjs +26 -91
- package/cli/companion/command.mjs +2 -0
- package/cli/demo-native.mjs +1 -1
- package/cli/parse.mjs +2 -1
- package/global/CHANGELOG.md +32 -0
- package/global/README.md +30 -102
- package/global/main.mjs +2 -0
- package/modules/container/adapters/d1-limits.mjs +2 -10
- package/modules/d1-transport/limits.d.mts +10 -0
- package/modules/d1-transport/limits.mjs +10 -0
- package/modules/demo-host/native/prepare.mjs +2 -1
- package/modules/deploy-kit/d1-import.mjs +11 -1
- package/modules/deploy-kit/migrations.mjs +14 -1
- package/modules/import-kit/schema/importers.schema.json +104 -2
- package/modules/import-kit/src/d1/README.md +14 -0
- package/modules/import-kit/src/d1/apply-reads.ts +74 -0
- package/modules/import-kit/src/d1/apply.ts +32 -9
- package/modules/import-kit/src/d1/history.ts +2 -2
- package/modules/import-kit/src/d1/redaction-step.ts +17 -7
- package/modules/import-kit/src/d1/types.ts +11 -0
- package/modules/import-kit/src/door/ADOPTING.md +0 -37
- package/modules/import-kit/src/door/README.md +0 -66
- package/modules/import-kit/src/door/admission.ts +3 -13
- package/modules/import-kit/src/door/authority.ts +0 -1
- package/modules/import-kit/src/door/failures.ts +1 -1
- package/modules/import-kit/src/door/grants.ts +5 -19
- package/modules/import-kit/src/door/handler.ts +1 -12
- package/modules/import-kit/src/door/index.ts +0 -6
- package/modules/import-kit/src/door/planning.ts +19 -44
- package/modules/import-kit/src/door/protocol.ts +2 -4
- package/modules/import-kit/src/door/schema.ts +0 -2
- package/modules/import-kit/src/door/store.ts +2 -2
- package/modules/import-kit/src/door/types.ts +1 -24
- package/modules/import-kit/src/provenance/credentials.ts +14 -5
- package/modules/import-kit/src/status/evidence.ts +3 -1
- package/modules/import-kit/src/status/index.ts +17 -2
- package/modules/import-kit/src/status/schema.ts +8 -0
- package/modules/import-kit/src/status/types.ts +25 -4
- package/package.json +1 -1
- package/cli/business/archive-task.mjs +0 -32
- package/cli/business/bounded-waves.d.mts +0 -23
- package/cli/business/bounded-waves.mjs +0 -57
- package/cli/business/catchup-task.mjs +0 -49
- package/cli/business/export-inputs.mjs +0 -79
- package/cli/business/export-task.mjs +0 -24
- package/cli/business/legacy-package.mjs +0 -117
- package/cli/business/package-transfer.mjs +0 -127
- package/cli/business/setup-demo.mjs +0 -95
- package/cli/business/vendor-door.mjs +0 -303
- package/cli/business/vendors/README.md +0 -215
- package/cli/business/vendors/calendly.mjs +0 -80
- package/cli/business/vendors/common.mjs +0 -280
- package/cli/business/vendors/credential-names.mjs +0 -124
- package/cli/business/vendors/index.mjs +0 -96
- package/cli/business/vendors/pagerduty.mjs +0 -61
- package/cli/business/vendors/sentry.mjs +0 -85
- package/cli/business/vendors/sorry.mjs +0 -106
- package/cli/business/vendors/uptimerobot.mjs +0 -21
- package/cli/business/vendors/zendesk-attachments.mjs +0 -57
- package/cli/business/vendors/zendesk-encode.mjs +0 -133
- package/cli/business/vendors/zendesk-guide.mjs +0 -34
- package/cli/business/vendors/zendesk-package.mjs +0 -28
- package/cli/business/vendors/zendesk-pages.mjs +0 -61
- package/cli/business/vendors/zendesk-people.mjs +0 -7
- package/cli/business/vendors/zendesk-store.mjs +0 -154
- package/cli/business/vendors/zendesk-tickets.mjs +0 -21
- package/cli/business/vendors/zendesk-token.mjs +0 -31
- package/cli/business/vendors/zendesk-transport.mjs +0 -48
- package/cli/business/vendors/zendesk.mjs +0 -61
- package/modules/import-kit/src/door/package.ts +0 -121
- package/modules/import-kit/src/door/waves/README.md +0 -158
- package/modules/import-kit/src/door/waves/digests.ts +0 -38
- package/modules/import-kit/src/door/waves/inventory.ts +0 -84
- package/modules/import-kit/src/door/waves/journal.ts +0 -19
- package/modules/import-kit/src/door/waves/lifecycle.ts +0 -242
- package/modules/import-kit/src/door/waves/preparation.ts +0 -190
- package/modules/import-kit/src/door/waves/schema.ts +0 -73
- package/modules/import-kit/src/door/waves/store.ts +0 -112
- package/modules/import-kit/src/door/waves/types.ts +0 -76
package/global/README.md
CHANGED
|
@@ -432,31 +432,24 @@ formats and syntax may yield no signals.
|
|
|
432
432
|
rer business setup --origin https://app.runeditrun.com --session-file /private/path/owner.session --confirm-demo
|
|
433
433
|
```
|
|
434
434
|
|
|
435
|
-
This
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
`
|
|
439
|
-
|
|
440
|
-
|
|
435
|
+
This creates the Business for the synthetic Juniper Harbor Software company in
|
|
436
|
+
`cli/business/setup-company.json` and saves its profile, through the platform's
|
|
437
|
+
owner routes (`GET` and `POST /api/business`, then `GET` and `PUT
|
|
438
|
+
/api/business/ID/profile`) using the signed-in person's session. It provisions
|
|
439
|
+
no seeds. The session file must be an owned regular file with mode `0600`,
|
|
440
|
+
containing only `rer_session=VALUE` or `__Host-rer_session=VALUE`. Symbolic links
|
|
441
|
+
are refused. The session is read in process, sent only as the platform Cookie
|
|
442
|
+
header, and never printed or placed in argv, local Business state or a receipt.
|
|
443
|
+
`--confirm-demo` is required.
|
|
441
444
|
|
|
442
445
|
Hosting defaults to `managed`; `--hosting assisted` and `--hosting self_host`
|
|
443
|
-
are also supported. There is no `hosted` alias. `--
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
receipt includes each seed's state and a top-level `complete` flag. Exit 0
|
|
451
|
-
requires `complete: true`: every required seed must be provisioned, token-issued,
|
|
452
|
-
policy-confirmed, settings-delivered and have its door proofs recorded.
|
|
453
|
-
Missing adoptions remain `not yet on the agent door` and exit nonzero.
|
|
454
|
-
|
|
455
|
-
The route order, policy, synthetic company profile and receipt are mirrored from
|
|
456
|
-
the RER platform at commit `0261033177e8e05284c037369ee9c3a85922a8aa`,
|
|
457
|
-
`scripts/business/demo.mjs` and `demo/company.json`. The platform owns the seed
|
|
458
|
-
catalogue, fixture data and door proofs. This command does not pair the computer
|
|
459
|
-
or use its stored device credential.
|
|
446
|
+
are also supported. There is no `hosted` alias. Both normal output and `--json`
|
|
447
|
+
print the receipt: `business_id`, `name`, `default_hosting`, `created` and
|
|
448
|
+
`profile` (`saved` or `unchanged`). Run it again to reuse the same Business: a
|
|
449
|
+
repeat whose profile matches changes nothing, and a Business of that name with
|
|
450
|
+
other hosting is refused (`existing_demo_hosting_differs`). Errors print a named
|
|
451
|
+
`{code}` to stderr and exit 1. This command does not pair the computer or use
|
|
452
|
+
its stored device credential.
|
|
460
453
|
|
|
461
454
|
## Business device commands
|
|
462
455
|
|
|
@@ -506,8 +499,7 @@ The install.sh form still passes either connect command through unchanged.
|
|
|
506
499
|
|
|
507
500
|
The owner chooses three things on the computer's row in the browser: "Add rules
|
|
508
501
|
so Claude Code never reads your secret files" (Claude Code user Read deny
|
|
509
|
-
rules), "Keep this computer awake while it works" (
|
|
510
|
-
prevention) and "Install the background service" (a launchd or systemd user
|
|
502
|
+
rules), "Keep this computer awake while it works" (recorded; nothing keeps the computer awake now that rer runs no exports) and "Install the background service" (a launchd or systemd user
|
|
511
503
|
service). Connect, run and next read them as `optIns` from device-authenticated
|
|
512
504
|
`GET /api/business/executors/self`, with keys `installPermissions`,
|
|
513
505
|
`awakeConsent` and `installService`; the CLI prints only the sentences. Only
|
|
@@ -540,29 +532,18 @@ refuses to remove the Keychain item (`security` exits 161 and leaves it), so
|
|
|
540
532
|
local credential is still stored" and names the one command that removes it when
|
|
541
533
|
run outside the sandbox; until then `status` says revoked, never paired.
|
|
542
534
|
|
|
543
|
-
`rer business vendor-key add sentry` requires hidden credential entry in a
|
|
544
|
-
terminal, stores it in the OS keychain and prints only `sentry:read`.
|
|
545
|
-
`remove` deletes it. The same commands support `uptimerobot`, `pagerduty` and
|
|
546
|
-
`calendly`. Sorry's export reads its public status page and needs no key. Never
|
|
547
|
-
put a key in command arguments, chat or a URL.
|
|
548
|
-
|
|
549
535
|
`rer business next` returns, within 60 seconds, one locally templated
|
|
550
536
|
classification, owner link or agent step, or says what `run` would start, and
|
|
551
|
-
says `Nothing yet` only when no agent step waits.
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
a `rollback.*` card, `switchoff.cancel`) are named as the owner's, with the
|
|
562
|
-
move's page (`/business/:id/moves/:toolId` when the summary names its
|
|
563
|
-
`tool_id`, else the switch-over board `/business/:id`), never as agent work. It
|
|
564
|
-
never reflects server instructions or raw evidence. Owner items cannot be
|
|
565
|
-
answered from this CLI.
|
|
537
|
+
says `Nothing yet` only when no agent step waits. A seed's **Move in** step is
|
|
538
|
+
the agent's step: `next` names the seed's Move in page, what the seed takes (a
|
|
539
|
+
file export, a read-only key or the old tool's public address), how to export
|
|
540
|
+
from the old tool and, for a tool the seed has no importer for, the seed's own
|
|
541
|
+
import format, all from the seed's `importers.json` row. The seed's Move in page
|
|
542
|
+
does the import (a dry run with counts, the import and undo) and rer imports
|
|
543
|
+
nothing itself; `rer business done ID --note "what moved"` finishes the step once
|
|
544
|
+
the data is in the seed. A step kind this rer does not know is reported by its
|
|
545
|
+
kind and never claimed or failed. It never reflects server instructions or raw
|
|
546
|
+
evidence. Owner items cannot be answered from this CLI.
|
|
566
547
|
`rer business answer ID VENDOR uses|sells|both|product-dependency [CONFIDENCE]`
|
|
567
548
|
submits a task-scoped classification through MCP. Choose a vendor ID from the
|
|
568
549
|
question's displayed answer schema, or `none`; the suggested vendor can be
|
|
@@ -574,12 +555,12 @@ a headers helper. That output is a short-lived credential: connect it directly
|
|
|
574
555
|
to the MCP client and do not paste it into an agent transcript.
|
|
575
556
|
|
|
576
557
|
`rer business status` shows local pairing, service state and progress without
|
|
577
|
-
printing checkpoints
|
|
558
|
+
printing checkpoints. Device credentials are stored separately
|
|
578
559
|
for each Business on an origin; disconnect also finds an older per-origin
|
|
579
560
|
credential through its original pairing record and revokes it once.
|
|
580
561
|
`rer business disconnect` revokes this
|
|
581
562
|
device through device-authenticated `DELETE /api/business/executors/self` before
|
|
582
|
-
removing its service, device
|
|
563
|
+
removing its service, device credential, any read-only key an earlier rer stored, state and introduced Claude
|
|
583
564
|
rules, preserving preexisting settings and unrelated edits. It says "Device
|
|
584
565
|
revoked" only after the server confirms (`{"revoked":true}`, or 401 for a device
|
|
585
566
|
already revoked). If this session may not remove the credential, a key, the
|
|
@@ -614,60 +595,7 @@ It removes `run.lock` on every exit, including Ctrl-C,
|
|
|
614
595
|
SIGTERM and a closed terminal (SIGHUP); a lock whose process is gone, or that
|
|
615
596
|
has not been refreshed for two minutes, is cleared on the next start.
|
|
616
597
|
|
|
617
|
-
|
|
618
|
-
import door. The claim's grant names the door (`/api/rer/import/uploads` on the
|
|
619
|
-
app's HTTPS origin) and `import_grants`; every door request asks
|
|
620
|
-
`POST /api/business/tasks/:taskId/import-grants` for its own short-lived
|
|
621
|
-
RER-signed grant (`uploads` or `plan`; apply is the server's alone after the
|
|
622
|
-
owner's wave approval), and the first grant's namespace names the vendor. The
|
|
623
|
-
read-only vendor key is read from OS credential storage, never from arguments,
|
|
624
|
-
the environment or a file. The shipped adapter runs into a private 0600
|
|
625
|
-
`export-<id>.bundle` file in the state folder, each item checked by the outbound
|
|
626
|
-
detector; the bundle is sent in 64 KiB chunks, then planned, which leaves a
|
|
627
|
-
zero-write preview in the door. Run then completes the task with outputs
|
|
628
|
-
`{upload_id, plan_id, bundle_sha256}` (identifiers only); Business reads that
|
|
629
|
-
plan from the door with its own grants and completes the task only if it is the
|
|
630
|
-
plan of exactly that bundle. Run prints only counts and removes the bundle on
|
|
631
|
-
every exit. Without the key, run
|
|
632
|
-
releases the task as `paused` after an `owner_required` event and names the
|
|
633
|
-
owner's command; a running supervisor picks the export up once the key is
|
|
634
|
-
stored. A door that is not ready releases it as `paused` after a
|
|
635
|
-
`connected_runner_required` event, naming the reason. UptimeRobot needs no
|
|
636
|
-
other input. Sentry, PagerDuty and Sorry use the leased task's `export_inputs`;
|
|
637
|
-
when those inputs are missing they release as `paused` after a
|
|
638
|
-
`configuration_required` event. Calendly's folder of files has no single-bundle
|
|
639
|
-
door format yet and releases with the same event. While an agent step waits (an export, or a
|
|
640
|
-
question for `next`, or the lease an earlier run on this computer still holds),
|
|
641
|
-
run says what it waits for instead of `Nothing yet`; otherwise it names a move's
|
|
642
|
-
owner step with its link before `Nothing yet`.
|
|
643
|
-
|
|
644
|
-
`rer business run` performs claimed `catchup.vendor` and `switchoff.archive`
|
|
645
|
-
tasks the same way: the claim's grant names the door, the key comes from OS
|
|
646
|
-
credential storage, and the shipped adapter's bundle goes up in 64 KiB chunks,
|
|
647
|
-
each with its own grant, from a private 0600 file removed on every exit.
|
|
648
|
-
|
|
649
|
-
- A catch-up then plans the upload with `{uploadId, catchup: true}`. Business
|
|
650
|
-
mints its plan grant under the owner's standing catch-up approval when one
|
|
651
|
-
covers the tool, and says so with `catchup: "bounded"` on the grant. If the
|
|
652
|
-
door refuses that with `catchup_requires_proposal:<reason>` (gone, conflicting
|
|
653
|
-
or rejected records, or a change of 5% or more), run asks
|
|
654
|
-
`import-grants` for `{action: "plan", review: true}` and plans the same upload
|
|
655
|
-
again for the owner's review. Run completes with
|
|
656
|
-
`{upload_id, plan_id, bundle_sha256}`; Business applies a bounded plan under
|
|
657
|
-
the standing approval and any other only after the owner approves it.
|
|
658
|
-
- An archive calls the door's `archive` action with `{uploadId, tool, takenAt}`
|
|
659
|
-
(the tool is the vendor's namespace), then `restore-check` with the
|
|
660
|
-
`archiveId`, and requires both answers to carry the bundle's SHA-256 and size,
|
|
661
|
-
the key `business-archive/<tool>/<takenAt>` and the same counts. Run completes
|
|
662
|
-
with `{archive_id, bundle_sha256}`; Business runs its own `restore-check`
|
|
663
|
-
before the task is done. Archive grants are `uploads`, `archive` and
|
|
664
|
-
`restore-check`; catch-up grants are `uploads`, `plan` and `status`.
|
|
665
|
-
|
|
666
|
-
Run prints only counts and the archive key. When an owner summary is supplied,
|
|
667
|
-
the CLI displays its browser link and refuses to answer it.
|
|
668
|
-
|
|
669
|
-
The five deterministic adapter implementations and their native output formats
|
|
670
|
-
are described in [the adapter guide](https://github.com/runeditrun/base/blob/main/cli/business/vendors/README.md).
|
|
598
|
+
Run prints only counts. When an owner summary is supplied, the CLI displays its browser link and refuses to answer it.
|
|
671
599
|
|
|
672
600
|
## Local engineering companion
|
|
673
601
|
|
package/global/main.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import { localDefaults } from './launch/local-defaults.mjs';
|
|
|
6
6
|
import { handleBrokenPipes } from './output.mjs';
|
|
7
7
|
import { repositoryCommands, repositoryHelp } from './cli/repository-contract.mjs';
|
|
8
8
|
import { repoArguments } from './cli/repository.mjs';
|
|
9
|
+
import { appSignInHelp } from '../cli/app-address.mjs';
|
|
9
10
|
|
|
10
11
|
const help = `Usage: rer VERB [FLAGS]
|
|
11
12
|
|
|
@@ -48,6 +49,7 @@ else its vendored scripts/rer.mjs.
|
|
|
48
49
|
of a checkout that pins the rer package use it).
|
|
49
50
|
|
|
50
51
|
Checkout verbs: ${Object.keys(checkoutVerbs).join(', ')}, listing. Run rer VERB --help in a checkout for its flags.
|
|
52
|
+
${appSignInHelp}
|
|
51
53
|
`;
|
|
52
54
|
|
|
53
55
|
export async function main(argv) {
|
|
@@ -1,13 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
* native D1 rejected a concatenation tree of depth 101 on 27 Oct 2026.
|
|
4
|
-
* Node 22.20.0 ignores DatabaseSync limits options: preflight is mandatory.
|
|
5
|
-
* Queries per Worker invocation (Paid 1000, Free 50, batch counts once) and
|
|
6
|
-
* the 30 second query deadline are NOT-ENFORCEABLE-LOCALLY here: a database
|
|
7
|
-
* connection has neither the invocation boundary nor the provider scheduler.
|
|
8
|
-
*/
|
|
9
|
-
export const d1NativeLimits = Object.freeze({ length: 2_000_000, sqlLength: 100_000,
|
|
10
|
-
column: 100, variableNumber: 100, functionArg: 32, likePatternLength: 50, exprDepth: 100 });
|
|
1
|
+
import { d1NativeLimits } from '../../d1-transport/limits.mjs';
|
|
2
|
+
export { d1NativeLimits } from '../../d1-transport/limits.mjs';
|
|
11
3
|
|
|
12
4
|
/** ENFORCED-BY-PREFLIGHT. Self-contained so workerd can run the same code.
|
|
13
5
|
* Depth is an estimate, not a complete SQLite parser. Parentheses, function
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** Shared native D1 ceilings used by SQL adapters and selected-key readers. */
|
|
2
|
+
export const d1NativeLimits: Readonly<{
|
|
3
|
+
length: number;
|
|
4
|
+
sqlLength: number;
|
|
5
|
+
column: number;
|
|
6
|
+
variableNumber: number;
|
|
7
|
+
functionArg: number;
|
|
8
|
+
likePatternLength: number;
|
|
9
|
+
exprDepth: number;
|
|
10
|
+
}>;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** Cloudflare D1 per-query limits, supplied from the platform limits page
|
|
2
|
+
* (updated 24 Oct 2026). Expression depth 100 is OBSERVED-NOT-DOCUMENTED:
|
|
3
|
+
* native D1 rejected a concatenation tree of depth 101 on 27 Oct 2026.
|
|
4
|
+
* Node 22.20.0 ignores DatabaseSync limits options: preflight is mandatory.
|
|
5
|
+
* Queries per Worker invocation (Paid 1000, Free 50, batch counts once) and
|
|
6
|
+
* the 30 second query deadline are NOT-ENFORCEABLE-LOCALLY here: a database
|
|
7
|
+
* connection has neither the invocation boundary nor the provider scheduler.
|
|
8
|
+
*/
|
|
9
|
+
export const d1NativeLimits = Object.freeze({ length: 2_000_000, sqlLength: 100_000,
|
|
10
|
+
column: 100, variableNumber: 100, functionArg: 32, likePatternLength: 50, exprDepth: 100 });
|
|
@@ -32,7 +32,8 @@ export async function prepareNative(folder) {
|
|
|
32
32
|
modules['_rer/isolate.js'] = { js: (await readFile(new URL('./isolate.mjs', import.meta.url), 'utf8')).replace('__RER_APP_ENTRY__', entry)
|
|
33
33
|
+ objectExports(bindings) };
|
|
34
34
|
for (const file of ISOLATE_FILES) modules[`_rer/${file}.js`] = { js: (await readFile(new URL(`./${file}.mjs`, import.meta.url), 'utf8')).replace('../../container/adapters/d1-limits.mjs', './d1-limits.js') };
|
|
35
|
-
modules['_rer/d1-limits.js'] = { js: await readFile(new URL('../../
|
|
35
|
+
modules['_rer/d1-native-limits.js'] = { js: await readFile(new URL('../../d1-transport/limits.mjs', import.meta.url), 'utf8') };
|
|
36
|
+
modules['_rer/d1-limits.js'] = { js: (await readFile(new URL('../../container/adapters/d1-limits.mjs', import.meta.url), 'utf8')).replaceAll('../../d1-transport/limits.mjs', './d1-native-limits.js') };
|
|
36
37
|
const assets = await collectFiles(join(app, 'dist/client'));
|
|
37
38
|
await cp(join(app, 'dist/client'), join(folder, 'assets'), { recursive: true });
|
|
38
39
|
const theme = await prepareTheme(folder, assets);
|
|
@@ -5,6 +5,10 @@ import { importDiagnostic } from './d1-import-diagnostics.mjs';
|
|
|
5
5
|
|
|
6
6
|
export const D1_IMPORT_POLL_LIMIT = 100;
|
|
7
7
|
|
|
8
|
+
// D1 answers a poll with this once the import it was running has finished: a
|
|
9
|
+
// short import can commit, completion marker included, before the next poll.
|
|
10
|
+
const NOT_IMPORTING = 'Not currently importing anything.';
|
|
11
|
+
|
|
8
12
|
// Wrangler's remote --file protocol. D1 imports execute the whole file atomically,
|
|
9
13
|
// including trigger bodies; the query endpoint's semicolon splitting cannot.
|
|
10
14
|
// Cloudflare's own reason for an import that failed or did not finish, kept on the
|
|
@@ -47,7 +51,13 @@ export async function importD1({ envelope, upload, redact = String }, id, sql) {
|
|
|
47
51
|
}
|
|
48
52
|
for (let polls = 0; polls < D1_IMPORT_POLL_LIMIT; polls++) {
|
|
49
53
|
// REST places errors on the envelope; validate nested errors when supplied.
|
|
50
|
-
if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length)))
|
|
54
|
+
if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length))) {
|
|
55
|
+
const failure = fail('d1-import-failed');
|
|
56
|
+
// Only a poll that follows an active import can mean it finished; the caller
|
|
57
|
+
// that knows the migration decides from the D1 ledger, never from this answer.
|
|
58
|
+
if (step === 'poll' && response?.success === false && response.error === NOT_IMPORTING && !response.errors?.length) failure.importNotActive = true;
|
|
59
|
+
throw failure;
|
|
60
|
+
}
|
|
51
61
|
if (response.status === 'complete') {
|
|
52
62
|
if (!(Number.isSafeInteger(response.result?.num_queries) && response.result.num_queries > 0 && typeof response.result.final_bookmark === 'string' && response.result.final_bookmark.length > 0)) throw fail('d1-import-invalid-result');
|
|
53
63
|
return;
|
|
@@ -53,6 +53,16 @@ export async function query(api, id, sql) {
|
|
|
53
53
|
assert(Array.isArray(rows) && rows.length > 0 && rows.every(x => x.success === true && Array.isArray(x.results)), 'd1-query-failed');
|
|
54
54
|
return rows;
|
|
55
55
|
}
|
|
56
|
+
/** D1 answered a poll "not currently importing" after an active ingest: the import may
|
|
57
|
+
* have committed before the poll. Complete only when the ledger holds this migration
|
|
58
|
+
* applied on exactly its bytes; any other answer, or a read that fails, leaves the
|
|
59
|
+
* import's own failure standing. */
|
|
60
|
+
async function ledgerAppliesMigration(api, id, migration) {
|
|
61
|
+
try {
|
|
62
|
+
const rows = (await query(api,id,`SELECT name, sha256, state FROM ${ledger} WHERE name=${literal(migration.name)}`))[0].results;
|
|
63
|
+
return rows.length === 1 && rows[0].state === 'applied' && rows[0].sha256 === migration.sha256;
|
|
64
|
+
} catch { return false; }
|
|
65
|
+
}
|
|
56
66
|
/** `locked`: the caller holds the install's deploy lock (owner.mjs withStateLock). */
|
|
57
67
|
export async function migrate(api, id, migrations, { apply = true, upgrade = apply, locked = false } = {}) {
|
|
58
68
|
validateMigrations(migrations);
|
|
@@ -80,7 +90,10 @@ export async function migrate(api, id, migrations, { apply = true, upgrade = app
|
|
|
80
90
|
? `UPDATE ${ledger} SET sha256=${literal(migration.sha256)}, state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(claimedFrom)} AND state='applying'`
|
|
81
91
|
: `UPDATE ${ledger} SET state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(migration.sha256)}`;
|
|
82
92
|
await api.importSql(id,`${migration.sql}\n${separator}${done};`)
|
|
83
|
-
.catch(error => {
|
|
93
|
+
.catch(async error => {
|
|
94
|
+
if (error?.importNotActive && await ledgerAppliesMigration(api,id,migration)) return;
|
|
95
|
+
error.migration ??= migration.name; throw error;
|
|
96
|
+
});
|
|
84
97
|
}
|
|
85
98
|
if (pending.length) {
|
|
86
99
|
// By name and bytes: rows applied after ones that sort above them keep their ids.
|
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
"required": ["$schema", "schemaVersion", "importers"],
|
|
7
7
|
"properties": {
|
|
8
8
|
"$schema": { "const": "./schema/importers.schema.json" },
|
|
9
|
-
"schemaVersion": { "
|
|
9
|
+
"schemaVersion": { "enum": [1, 2] },
|
|
10
|
+
"format": { "$ref": "#/$defs/format" },
|
|
10
11
|
"importers": {
|
|
11
12
|
"type": "array",
|
|
12
13
|
"minItems": 1,
|
|
@@ -14,13 +15,21 @@
|
|
|
14
15
|
"oneOf": [
|
|
15
16
|
{ "$ref": "#/$defs/verifiedImporter" },
|
|
16
17
|
{ "$ref": "#/$defs/documentedImporter" },
|
|
17
|
-
{ "$ref": "#/$defs/noneImporter" }
|
|
18
|
+
{ "$ref": "#/$defs/noneImporter" },
|
|
19
|
+
{ "$ref": "#/$defs/moveIn" },
|
|
20
|
+
{ "$ref": "#/$defs/moveInVerifiedImporter" },
|
|
21
|
+
{ "$ref": "#/$defs/moveInDocumentedImporter" },
|
|
22
|
+
{ "$ref": "#/$defs/moveInNoneImporter" }
|
|
18
23
|
]
|
|
19
24
|
}
|
|
20
25
|
}
|
|
21
26
|
},
|
|
22
27
|
"additionalProperties": false,
|
|
23
28
|
"$defs": {
|
|
29
|
+
"format": { "type": "string", "minLength": 1, "maxLength": 300, "pattern": "^(?!.*(?:^|/)\\.\\.(?:/|$))[A-Za-z0-9._][A-Za-z0-9._/-]*$" },
|
|
30
|
+
"moveInPath": { "type": "string", "minLength": 1, "maxLength": 300, "pattern": "^/(?:[^/\\s][^\\s]*)?$" },
|
|
31
|
+
"accepts": { "type": "array", "minItems": 1, "maxItems": 3, "uniqueItems": true, "items": { "enum": ["file", "key", "address"] } },
|
|
32
|
+
"export": { "type": "string", "minLength": 1, "maxLength": 1000, "pattern": "^[^\\r\\n]+$" },
|
|
24
33
|
"artifact": {
|
|
25
34
|
"type": "object",
|
|
26
35
|
"required": ["link", "sha256", "size"],
|
|
@@ -102,6 +111,99 @@
|
|
|
102
111
|
}
|
|
103
112
|
},
|
|
104
113
|
"additionalProperties": false
|
|
114
|
+
},
|
|
115
|
+
"moveIn": {
|
|
116
|
+
"type": "object",
|
|
117
|
+
"required": ["incumbent", "product", "moveIn", "accepts", "export"],
|
|
118
|
+
"properties": {
|
|
119
|
+
"incumbent": { "type": "string", "minLength": 1 },
|
|
120
|
+
"product": { "type": "string", "minLength": 1 },
|
|
121
|
+
"moveIn": { "$ref": "#/$defs/moveInPath" },
|
|
122
|
+
"accepts": { "$ref": "#/$defs/accepts" },
|
|
123
|
+
"export": { "$ref": "#/$defs/export" }
|
|
124
|
+
},
|
|
125
|
+
"additionalProperties": false
|
|
126
|
+
},
|
|
127
|
+
"moveInVerifiedImporter": {
|
|
128
|
+
"type": "object",
|
|
129
|
+
"required": ["incumbent", "product", "moveIn", "accepts", "export", "status", "evidence"],
|
|
130
|
+
"properties": {
|
|
131
|
+
"incumbent": { "type": "string", "minLength": 1 },
|
|
132
|
+
"product": { "type": "string", "minLength": 1 },
|
|
133
|
+
"moveIn": { "$ref": "#/$defs/moveInPath" },
|
|
134
|
+
"accepts": { "$ref": "#/$defs/accepts" },
|
|
135
|
+
"export": { "$ref": "#/$defs/export" },
|
|
136
|
+
"status": { "const": "verified" },
|
|
137
|
+
"evidence": {
|
|
138
|
+
"type": "object",
|
|
139
|
+
"required": ["link", "sha256", "size", "research"],
|
|
140
|
+
"properties": {
|
|
141
|
+
"link": { "type": "string", "minLength": 1 },
|
|
142
|
+
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
143
|
+
"size": { "type": "integer", "minimum": 1 },
|
|
144
|
+
"research": { "$ref": "#/$defs/artifact" }
|
|
145
|
+
},
|
|
146
|
+
"additionalProperties": false
|
|
147
|
+
}
|
|
148
|
+
},
|
|
149
|
+
"additionalProperties": false
|
|
150
|
+
},
|
|
151
|
+
"moveInDocumentedImporter": {
|
|
152
|
+
"type": "object",
|
|
153
|
+
"required": ["incumbent", "product", "moveIn", "accepts", "export", "status", "evidence"],
|
|
154
|
+
"properties": {
|
|
155
|
+
"incumbent": { "type": "string", "minLength": 1 },
|
|
156
|
+
"product": { "type": "string", "minLength": 1 },
|
|
157
|
+
"moveIn": { "$ref": "#/$defs/moveInPath" },
|
|
158
|
+
"accepts": { "$ref": "#/$defs/accepts" },
|
|
159
|
+
"export": { "$ref": "#/$defs/export" },
|
|
160
|
+
"status": { "const": "documented format" },
|
|
161
|
+
"evidence": {
|
|
162
|
+
"type": "object",
|
|
163
|
+
"required": [
|
|
164
|
+
"link", "sha256", "size", "sourceRepository", "sourceRevision",
|
|
165
|
+
"sourcePath", "sourceSha256", "validation", "realExport"
|
|
166
|
+
],
|
|
167
|
+
"properties": {
|
|
168
|
+
"link": { "type": "string", "minLength": 1 },
|
|
169
|
+
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
170
|
+
"size": { "type": "integer", "minimum": 1 },
|
|
171
|
+
"sourceRepository": { "type": "string", "minLength": 1 },
|
|
172
|
+
"sourceRevision": { "type": "string", "pattern": "^[a-f0-9]{40,64}$" },
|
|
173
|
+
"sourcePath": { "type": "string", "minLength": 1 },
|
|
174
|
+
"sourceSha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
175
|
+
"validation": { "const": "synthetic-only" },
|
|
176
|
+
"realExport": { "const": "not-supplied" }
|
|
177
|
+
},
|
|
178
|
+
"additionalProperties": false
|
|
179
|
+
}
|
|
180
|
+
},
|
|
181
|
+
"additionalProperties": false
|
|
182
|
+
},
|
|
183
|
+
"moveInNoneImporter": {
|
|
184
|
+
"type": "object",
|
|
185
|
+
"required": ["incumbent", "product", "moveIn", "accepts", "export", "status", "evidence"],
|
|
186
|
+
"properties": {
|
|
187
|
+
"incumbent": { "type": "string", "minLength": 1 },
|
|
188
|
+
"product": { "type": "string", "minLength": 1 },
|
|
189
|
+
"moveIn": { "$ref": "#/$defs/moveInPath" },
|
|
190
|
+
"accepts": { "$ref": "#/$defs/accepts" },
|
|
191
|
+
"export": { "$ref": "#/$defs/export" },
|
|
192
|
+
"status": { "const": "none" },
|
|
193
|
+
"evidence": {
|
|
194
|
+
"type": "object",
|
|
195
|
+
"required": ["link", "sha256", "size", "reason", "checkedAt"],
|
|
196
|
+
"properties": {
|
|
197
|
+
"link": { "type": "string", "minLength": 1 },
|
|
198
|
+
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
|
199
|
+
"size": { "type": "integer", "minimum": 1 },
|
|
200
|
+
"reason": { "type": "string", "minLength": 1 },
|
|
201
|
+
"checkedAt": { "type": "string", "format": "date-time" }
|
|
202
|
+
},
|
|
203
|
+
"additionalProperties": false
|
|
204
|
+
}
|
|
205
|
+
},
|
|
206
|
+
"additionalProperties": false
|
|
105
207
|
}
|
|
106
208
|
}
|
|
107
209
|
}
|
|
@@ -153,6 +153,15 @@ are local runtime receipts, not hosted capacity evidence.
|
|
|
153
153
|
The product implements `D1TargetAdapter`:
|
|
154
154
|
|
|
155
155
|
- `read(database, destination, kind, id)` returns the complete native `TargetRow`.
|
|
156
|
+
- Optional `readMany(database, destination, keys)` returns the same complete rows
|
|
157
|
+
as individual reads, in key order, including duplicate keys and missing rows.
|
|
158
|
+
Query only selected keys and preserve native decoding. With private transport,
|
|
159
|
+
adapters that implement `stagingRead` can also provide `stagingReadMany`, with
|
|
160
|
+
the same exact before-read guards for each key. Missing bulk methods retain
|
|
161
|
+
individual reads; any bulk-read error except request-budget exhaustion discards the
|
|
162
|
+
snapshot and uses that path. Exhaustion ends the batch like the budget reserve does.
|
|
163
|
+
An inherited bulk method must preserve an overridden single read or throw.
|
|
164
|
+
Bulk observations belong only to the current publication.
|
|
156
165
|
- `matches(rowOrNull, destination, kind, id)` returns an exact SQL boolean snapshot
|
|
157
166
|
predicate covering the native values, revision, attachment and related evidence,
|
|
158
167
|
or complete absence for null. This is the optimistic concurrency fence. Checking
|
|
@@ -194,6 +203,11 @@ per request. The port does not retry failed SQL, conflicts or revoked authority.
|
|
|
194
203
|
A failed batch leaves its prior committed checkpoint intact; the product can fix
|
|
195
204
|
the condition and explicitly call `resumeD1Import`, or `rollbackD1Import`.
|
|
196
205
|
|
|
206
|
+
Saved read budget can advance a budget-limited call's checkpoint or surface a
|
|
207
|
+
later entry's refusal one call earlier, with the same error code and text.
|
|
208
|
+
Such a refusal publishes none of the gathered entries and leaves the job at its
|
|
209
|
+
last consistent durable checkpoint.
|
|
210
|
+
|
|
197
211
|
Each batch guards the immutable plan, actor binding, checkpoint and durable fence
|
|
198
212
|
inside the same D1 transaction as native rows, originals, source ownership,
|
|
199
213
|
provenance and journal. Successful batches increment the fence. Unlike Node's
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { d1NativeLimits } from '../../../d1-transport/limits.mjs';
|
|
2
|
+
import type { PlanEntry, TargetRow } from '../types.ts';
|
|
3
|
+
import { D1RequestLimitError } from './budget.ts';
|
|
4
|
+
import type { D1ImportContext, D1Sql } from './types.ts';
|
|
5
|
+
|
|
6
|
+
const key = (kind: string, id: string): string => JSON.stringify([kind, id]);
|
|
7
|
+
export interface ApplyReads {
|
|
8
|
+
sources: Map<string, any>;
|
|
9
|
+
predecessors: Map<string, any>;
|
|
10
|
+
targets: Map<string, { row: TargetRow | undefined; guards?: D1Sql[] }>;
|
|
11
|
+
}
|
|
12
|
+
export const targetReadKey = key;
|
|
13
|
+
|
|
14
|
+
const emptyReads = (): ApplyReads => ({ sources: new Map(), predecessors: new Map(), targets: new Map() });
|
|
15
|
+
const providerQueries = Math.floor((d1NativeLimits.variableNumber - 1) / 2);
|
|
16
|
+
|
|
17
|
+
/** How far the next bulk read may reach: the end (exclusive) of the entries from
|
|
18
|
+
* `from` that can still publish in this call. It stops at the 1000-effect batch
|
|
19
|
+
* cap, and where the reads, counted as the contract's worst case of one query per
|
|
20
|
+
* target key, would take more than a quarter of what the request has left after the
|
|
21
|
+
* publication so far. The entry at `from` is always included; later windows are
|
|
22
|
+
* sized again against the budget those reads actually used. */
|
|
23
|
+
export function readEnd(pending: PlanEntry[], from: number, gathered: number, budget: number, reads?: ApplyReads): number {
|
|
24
|
+
const identities = new Set<string>(), keys = new Set<string>();
|
|
25
|
+
let effects = 0, end = from;
|
|
26
|
+
for (; end < pending.length; end++) {
|
|
27
|
+
const entry = pending[end];
|
|
28
|
+
if (end > from && gathered + effects + entry.effects.length > 1000) break;
|
|
29
|
+
identities.add(entry.identity);
|
|
30
|
+
for (const effect of entry.effects) { const k = key(effect.targetKind, effect.targetId); if (!reads?.targets.has(k)) keys.add(k); }
|
|
31
|
+
const queries = Math.ceil(identities.size / d1NativeLimits.variableNumber) + keys.size + Math.ceil(keys.size / providerQueries);
|
|
32
|
+
if (end > from && queries > budget / 4) break;
|
|
33
|
+
effects += entry.effects.length;
|
|
34
|
+
}
|
|
35
|
+
return end;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Reads this window's rows into `reads`, which holds only this publication's
|
|
39
|
+
* observations; no observation survives atomic(). A budget exhaustion propagates
|
|
40
|
+
* so the gather can stop the way its own reserve does. Any other failure discards
|
|
41
|
+
* the entire snapshot, including any earlier successful window. */
|
|
42
|
+
export async function applyReads(entries: PlanEntry[], context: D1ImportContext, reads: ApplyReads = emptyReads()): Promise<ApplyReads | undefined> {
|
|
43
|
+
const { database: db, targets } = context.storage;
|
|
44
|
+
const staged = context.maxBatchBytes !== undefined && !!targets.stagingRead;
|
|
45
|
+
if (staged ? !targets.stagingReadMany : !targets.readMany) return;
|
|
46
|
+
try {
|
|
47
|
+
const { sources, predecessors, targets: native } = reads;
|
|
48
|
+
const identities = [...new Set(entries.map(entry => entry.identity))];
|
|
49
|
+
for (let i = 0; i < identities.length; i += d1NativeLimits.variableNumber) {
|
|
50
|
+
const part = identities.slice(i, i + d1NativeLimits.variableNumber);
|
|
51
|
+
const { results } = await db.prepare(`SELECT * FROM ik_sources WHERE identity IN (${part.map(() => '?').join(',')})`).bind(...part).all();
|
|
52
|
+
for (const row of results) sources.set(row.identity, row);
|
|
53
|
+
}
|
|
54
|
+
const keys = [...new Map(entries.flatMap(entry => entry.effects.map(effect => [key(effect.targetKind, effect.targetId), { kind: effect.targetKind, id: effect.targetId }] as const)).filter(([k]) => !native.has(k))).values()];
|
|
55
|
+
const snapshots: { row: TargetRow | undefined; guards?: D1Sql[] }[] = !keys.length ? [] : staged
|
|
56
|
+
? await targets.stagingReadMany!(db, context.destination, keys)
|
|
57
|
+
: (await targets.readMany!(db, context.destination, keys)).map(row => ({ row }));
|
|
58
|
+
if (snapshots.length !== keys.length) throw new Error('IMPORT_NATIVE_STAGING_GUARD_REQUIRED');
|
|
59
|
+
for (const [i, selected] of keys.entries()) {
|
|
60
|
+
const snapshot = snapshots[i];
|
|
61
|
+
if (staged && !snapshot.guards?.length) throw new Error('IMPORT_NATIVE_STAGING_GUARD_REQUIRED');
|
|
62
|
+
native.set(key(selected.kind, selected.id), snapshot);
|
|
63
|
+
}
|
|
64
|
+
for (let i = 0; i < keys.length; i += providerQueries) {
|
|
65
|
+
const part = keys.slice(i, i + providerQueries);
|
|
66
|
+
const { results } = await db.prepare(`SELECT * FROM ik_provenance WHERE destination=? AND (target_kind,target_id) IN (${part.map(() => '(?,?)').join(',')}) AND reversed=0`).bind(context.destination, ...part.flatMap(k => [k.kind, k.id])).all();
|
|
67
|
+
for (const row of results) predecessors.set(key(row.target_kind, row.target_id), row);
|
|
68
|
+
}
|
|
69
|
+
return reads;
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (error instanceof D1RequestLimitError) throw error;
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
}
|