@runeditrun/rer 0.2.99 → 0.2.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/README.md +88 -2
- package/cli/business/bounded-waves.d.mts +23 -0
- package/cli/business/bounded-waves.mjs +57 -0
- package/cli/business/catchup-task.mjs +9 -2
- package/cli/business/command.mjs +8 -3
- package/cli/business/export-inputs.mjs +12 -2
- package/cli/business/export-task.mjs +9 -2
- package/cli/business/legacy-package.mjs +117 -0
- package/cli/business/local-commands.mjs +10 -1
- package/cli/business/package-transfer.mjs +127 -0
- package/cli/business/state.mjs +3 -3
- package/cli/business/vendor-door.mjs +83 -11
- package/cli/business/vendors/README.md +41 -2
- package/cli/business/vendors/common.mjs +81 -0
- package/cli/business/vendors/index.mjs +12 -2
- package/cli/business/vendors/zendesk-attachments.mjs +57 -0
- package/cli/business/vendors/zendesk-encode.mjs +133 -0
- package/cli/business/vendors/zendesk-guide.mjs +34 -0
- package/cli/business/vendors/zendesk-package.mjs +28 -0
- package/cli/business/vendors/zendesk-pages.mjs +61 -0
- package/cli/business/vendors/zendesk-people.mjs +7 -0
- package/cli/business/vendors/zendesk-store.mjs +154 -0
- package/cli/business/vendors/zendesk-tickets.mjs +21 -0
- package/cli/business/vendors/zendesk-token.mjs +31 -0
- package/cli/business/vendors/zendesk-transport.mjs +48 -0
- package/cli/business/vendors/zendesk.mjs +61 -0
- package/cli/deploy-command.mjs +5 -4
- package/cli/deploy-inputs.mjs +1 -1
- package/cli/deploy.mjs +6 -4
- package/cli/production-auth.mjs +16 -3
- package/cli/production-mail.mjs +18 -1
- package/cli/site.mjs +1 -1
- package/global/CHANGELOG.md +21 -0
- package/modules/deploy-kit/README.md +10 -0
- package/modules/deploy-kit/provider.mjs +18 -2
- package/modules/import-kit/src/d1/jobs.ts +3 -1
- package/modules/import-kit/src/d1/retirement-scope.ts +2 -2
- package/modules/import-kit/src/d1/retirement-tally.ts +1 -0
- package/modules/import-kit/src/d1/retirement.ts +15 -5
- package/modules/import-kit/src/d1/types.ts +3 -0
- package/modules/import-kit/src/d1/wave-boundary.ts +33 -0
- package/modules/import-kit/src/d1/wave-retirement.ts +47 -0
- package/modules/import-kit/src/door/ADOPTING.md +37 -0
- package/modules/import-kit/src/door/README.md +66 -0
- package/modules/import-kit/src/door/admission.ts +13 -3
- package/modules/import-kit/src/door/authority.ts +1 -0
- package/modules/import-kit/src/door/failures.ts +1 -1
- package/modules/import-kit/src/door/grants.ts +19 -5
- package/modules/import-kit/src/door/handler.ts +12 -1
- package/modules/import-kit/src/door/index.ts +6 -0
- package/modules/import-kit/src/door/package.ts +121 -0
- package/modules/import-kit/src/door/planning.ts +44 -19
- package/modules/import-kit/src/door/protocol.ts +4 -2
- package/modules/import-kit/src/door/schema.ts +2 -0
- package/modules/import-kit/src/door/store.ts +2 -2
- package/modules/import-kit/src/door/types.ts +24 -1
- package/modules/import-kit/src/door/waves/README.md +158 -0
- package/modules/import-kit/src/door/waves/digests.ts +38 -0
- package/modules/import-kit/src/door/waves/inventory.ts +84 -0
- package/modules/import-kit/src/door/waves/journal.ts +19 -0
- package/modules/import-kit/src/door/waves/lifecycle.ts +242 -0
- package/modules/import-kit/src/door/waves/preparation.ts +190 -0
- package/modules/import-kit/src/door/waves/schema.ts +73 -0
- package/modules/import-kit/src/door/waves/store.ts +112 -0
- package/modules/import-kit/src/door/waves/types.ts +76 -0
- package/onboarding/adapter.mjs +7 -1
- package/onboarding/operator-installer.mjs +3 -1
- package/package.json +1 -1
- package/scripts/publish-check/README.md +20 -0
- package/scripts/publish-check/diagnostics.mjs +73 -0
- package/scripts/publish-check/slots.py +37 -7
- package/scripts/publish-check.mjs +64 -37
package/cli/README.md
CHANGED
|
@@ -70,8 +70,13 @@ its port over their private parent channel. The CLI confirms that the runtime
|
|
|
70
70
|
owns that socket, then updates the traffic proxy and scheduled-trigger target.
|
|
71
71
|
Custom runtime targets retain the assigned internal port contract: bind it
|
|
72
72
|
strictly. A healthy application on a different owned port causes an immediate
|
|
73
|
-
port-mismatch refusal.
|
|
74
|
-
|
|
73
|
+
port-mismatch refusal. A start target can declare a finite positive `timeoutMs`
|
|
74
|
+
for listener announcement and health together; targets without it retain the
|
|
75
|
+
60-second default. The shared development target used by setup and dev declares
|
|
76
|
+
fifteen minutes for real Vite startup. A missing listener announcement reports
|
|
77
|
+
elapsed and allowed time and points to the runtime startup log; auxiliary
|
|
78
|
+
process sockets cannot satisfy readiness. An individual probe gets at most five
|
|
79
|
+
seconds by default. Concurrent
|
|
75
80
|
starts waiting on an existing server have one five-minute elapsed deadline,
|
|
76
81
|
including control requests and polling. Loopback HTTP requests use an elapsed
|
|
77
82
|
deadline, so a response that keeps sending bytes cannot extend readiness.
|
|
@@ -272,6 +277,21 @@ secrets after the provider verifies the owned deployment. Local setup and demo
|
|
|
272
277
|
do not require production mail credentials. Preview serves production output
|
|
273
278
|
and prints its ready address without a captured-mail sign-in flow.
|
|
274
279
|
|
|
280
|
+
A deployment can instead turn outbound email off on purpose. `MAIL_PROVIDER=none`
|
|
281
|
+
in the environment or the `--env-file` is accepted only when the seed's adapter
|
|
282
|
+
declares a sign-in that works without mail: the `commandAdapter` option
|
|
283
|
+
`mailOffSignIn`, one line naming the command a person runs from the checkout. Both
|
|
284
|
+
`rer deploy` and `rer deploy --dry-run` then pass the production sign-in check
|
|
285
|
+
without any mail input, bind nothing that can send, and report that outbound email
|
|
286
|
+
is off together with that command. `none` is never inferred: a missing
|
|
287
|
+
`MAIL_PROVIDER`, `later` or `direct` keeps today's refusal, a seed without the
|
|
288
|
+
declaration keeps today's refusal for `none` too, and `none` together with a
|
|
289
|
+
mail credential (`MAIL_API_KEY`, `SENDGRID_API_KEY`, `RESEND_API_KEY`,
|
|
290
|
+
`MAIL_PROVIDER_API_KEY`, `MAIL_ACCOUNT_ID` or `MAIL_RELAY_URL`) refuses before
|
|
291
|
+
anything is built or uploaded. The seed owns what an installation does with mail
|
|
292
|
+
off, and turning mail on later is a `rer deploy --set-secret` of the provider
|
|
293
|
+
inputs the seed documents.
|
|
294
|
+
|
|
275
295
|
The production preflight also reads the product's `artifact.specInputs` in
|
|
276
296
|
`rer-packaging.json`, `packaging.json` or `scripts/delivery/base/packaging.json`,
|
|
277
297
|
and `management/integration/deployment-spec.json`. Public inputs come from
|
|
@@ -602,3 +622,69 @@ new clauses start missing, and seed-owned AGENT and SURFACE clauses stay intact.
|
|
|
602
622
|
Run the edition's source-lock command last. See
|
|
603
623
|
[Contract clauses](../modules/agent-door/ADOPTING.md#contract-clauses) for source
|
|
604
624
|
delivery and generation details.
|
|
625
|
+
|
|
626
|
+
## Zendesk offline snapshots
|
|
627
|
+
|
|
628
|
+
The owner creates one confidential OAuth client in Zendesk Admin Center, owned
|
|
629
|
+
by an admin (incremental exports require admin access), whose allowed scopes are
|
|
630
|
+
read-only. Allow `tickets:read users:read organizations:read hc:read
|
|
631
|
+
ticket_attachments:read`, or the account's read-only `read` scope. Do not allow
|
|
632
|
+
write scopes. In the same sitting, on the paired computer, run:
|
|
633
|
+
|
|
634
|
+
```sh
|
|
635
|
+
rer business vendor-key add zendesk
|
|
636
|
+
```
|
|
637
|
+
|
|
638
|
+
At the hidden prompt enter one line of JSON with `client_id` and
|
|
639
|
+
`client_secret`. Never put it in an argument, environment variable, chat or
|
|
640
|
+
Business. macOS Keychain or Linux Secret Service keeps it as
|
|
641
|
+
`vendor:zendesk:read`; no file fallback exists. Each run mints a 1,800-second
|
|
642
|
+
client-credentials token and checks its current scopes before requesting data.
|
|
643
|
+
Only this OAuth mint uses POST. Every data request is GET. The CLI never changes
|
|
644
|
+
Zendesk. A 401/403 pauses for the existing key-replacement step; rerunning resumes
|
|
645
|
+
acknowledged pages with a new short-lived token.
|
|
646
|
+
|
|
647
|
+
Business supplies `{account:{subdomain},mode,highWater}` to the leased task.
|
|
648
|
+
`mode` defaults to `snapshot`. `sweep` requires the retained snapshot's numeric
|
|
649
|
+
`highWater`, reads incremental tickets after it, refreshes their comments and
|
|
650
|
+
attachments, and delivers a complete snapshot including retained older records.
|
|
651
|
+
`census` reads only incremental tickets from zero, counts unique ids and tickets
|
|
652
|
+
newer than `highWater`, and saves `rer-zendesk-census/1` locally. It never uploads
|
|
653
|
+
a partial snapshot. Business census task outputs remain a separate adoption:
|
|
654
|
+
the CLI reports `CENSUS_OUTPUTS_UNSUPPORTED` and pauses after the local result.
|
|
655
|
+
|
|
656
|
+
Raw pages, provenance, deduplicated SHA-256 blobs, a constant-size checkpoint,
|
|
657
|
+
transfer metadata and the census live in a private `zendesk-<business/account
|
|
658
|
+
hash>` directory under the Business state directory. The snapshot survives task
|
|
659
|
+
handover, upload refusal and process death until retire; a new computer starts
|
|
660
|
+
from zero. Do not edit its objects: every referenced hash and size is verified
|
|
661
|
+
on resume before any network request. Keep this private customer data on an
|
|
662
|
+
encrypted disk with the same custody as an authorized Zendesk export.
|
|
663
|
+
|
|
664
|
+
A `retire` input deletes the Zendesk key and all retained Zendesk stores, records
|
|
665
|
+
local revocation, and refuses later exports. It never calls a provider revoke or
|
|
666
|
+
write endpoint; outstanding short-lived tokens expire. Business's retire task
|
|
667
|
+
acknowledgement is a separate adoption (`RETIRE_OUTPUTS_UNSUPPORTED`).
|
|
668
|
+
`rer business vendor-key remove zendesk` performs the same local revocation;
|
|
669
|
+
`disconnect` also removes the retained snapshots. Do not retire before the mail
|
|
670
|
+
rollback window and Zendesk retire decision.
|
|
671
|
+
|
|
672
|
+
The destination receives wrapped HH `happytohelp-zendesk-offline-v1` NDJSON.
|
|
673
|
+
Archived tickets are included, deleted tickets remain census tombstones, private
|
|
674
|
+
comments remain private, and source reply-routing and restriction fields stay.
|
|
675
|
+
Credential-shaped content is deterministically removed from the delivered copy
|
|
676
|
+
and listed by record kind, id and JSON pointer in the local census. Raw data
|
|
677
|
+
stays private locally. URLs lose access query/fragment/userinfo; attachment
|
|
678
|
+
access URL fields are removed from the delivered copy. Unavailable attachments
|
|
679
|
+
are counted with fixed reasons; no binding is invented.
|
|
680
|
+
|
|
681
|
+
A whole snapshot fitting 2 MiB, 5,000 records, 128 KiB per ticket, and at most
|
|
682
|
+
20 inline attachments of 48 KiB per ticket goes as one v1 bundle. Anything larger
|
|
683
|
+
uses the shared logical package sender with path bindings, deduplicated blobs,
|
|
684
|
+
census, tombstones and Guide sidecars. The manifest probes the destination
|
|
685
|
+
before segments are sent. A destination without the package-aware reader answers
|
|
686
|
+
`SEED_PACKAGE_UNSUPPORTED`: the complete local snapshot stays and no segments
|
|
687
|
+
are uploaded. Never narrow the snapshot to fit. Translations, categories, sections,
|
|
688
|
+
segments, article media and tombstones are retained as package sidecars and
|
|
689
|
+
counted pending HH adoption. This is a synthetic-edge implementation, not proof
|
|
690
|
+
of real-account parity, client-scope availability or a completed migration.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { WaveTotals, WaveBinding } from '../../modules/import-kit/src/door/waves/types.ts';
|
|
2
|
+
export interface BoundedApproval { id: string; at: string }
|
|
3
|
+
export interface BoundedArrangement {
|
|
4
|
+
protocol: 'rer-bounded-waves/1'; migration: string; revision: string; policy: string;
|
|
5
|
+
catchup?: boolean; parentId?: string;
|
|
6
|
+
parentApproval?: BoundedApproval & { fingerprint: string };
|
|
7
|
+
children?: { ordinal: number; fingerprint: string; approval?: BoundedApproval }[];
|
|
8
|
+
}
|
|
9
|
+
export interface BoundedAggregate {
|
|
10
|
+
protocol: 'rer-bounded-waves/1'; parentId: string; fingerprint: string;
|
|
11
|
+
aggregateSuccess: true; nativeSuccess: boolean; completeHead: string; counts: WaveTotals;
|
|
12
|
+
goneButRetained: { kind: string; sourceId: string }[]; children: number;
|
|
13
|
+
}
|
|
14
|
+
export interface BoundedPending {
|
|
15
|
+
parentId: string; state: 'awaiting_parent_approval' | 'awaiting_child_approval' | 'partial';
|
|
16
|
+
aggregateSuccess: false; fingerprint?: string; ordinal?: number; children?: unknown[];
|
|
17
|
+
}
|
|
18
|
+
export function sendBoundedWaves(options: {
|
|
19
|
+
arrangement: BoundedArrangement; uploadId: string; signal: AbortSignal;
|
|
20
|
+
fresh(action: string, extra: { bounded: WaveBinding }): Promise<unknown>;
|
|
21
|
+
request(grant: unknown, body: Record<string, unknown>, signal: AbortSignal): Promise<any>;
|
|
22
|
+
checkpoint?(progress: Record<string, string | number | boolean | undefined>): Promise<void>;
|
|
23
|
+
}): Promise<BoundedAggregate | BoundedPending>;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
const PROTOCOL = 'rer-bounded-waves/1';
|
|
2
|
+
const digest = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value);
|
|
3
|
+
const id = value => typeof value === 'string' && /^[A-Za-z0-9_-]{1,128}$/.test(value);
|
|
4
|
+
/** Business supplies a typed arrangement and fresh grants. The sender cannot
|
|
5
|
+
* choose a destination or manufacture any parent/child approval. */
|
|
6
|
+
export async function sendBoundedWaves({ arrangement, uploadId, fresh, request, signal, checkpoint = async () => {} }) {
|
|
7
|
+
if (arrangement?.protocol !== PROTOCOL || !['migration','revision','policy'].every(k => id(arrangement[k]))) throw Error('BOUNDED_TASK_REQUIRED: Business must supply the typed bounded-wave arrangement.');
|
|
8
|
+
const binding = { protocol: PROTOCOL, migration: arrangement.migration, revision: arrangement.revision, policy: arrangement.policy };
|
|
9
|
+
const post = async (action, body, extra = {}) => {
|
|
10
|
+
signal.throwIfAborted();
|
|
11
|
+
const grant = await fresh(action, { bounded: { ...binding, ...extra } });
|
|
12
|
+
return request(grant, body, signal);
|
|
13
|
+
};
|
|
14
|
+
let parentId = arrangement.parentId;
|
|
15
|
+
if (parentId !== undefined && !id(parentId)) throw Error('Invalid bounded parent identifier.');
|
|
16
|
+
// Replaying page zero resolves the durable preparation cursor, including a
|
|
17
|
+
// response lost before Business saved the latest page checkpoint.
|
|
18
|
+
{
|
|
19
|
+
let ordinal = 0;
|
|
20
|
+
for (;;) {
|
|
21
|
+
const prepared = await post('prepare', { ...(parentId ? { parentId } : { uploadId, ...(arrangement.catchup ? { catchup: true } : {}) }), ordinal });
|
|
22
|
+
if (!id(prepared.parentId) || !Number.isSafeInteger(prepared.next) || prepared.next <= ordinal) throw Error('Invalid bounded preparation response.');
|
|
23
|
+
parentId = prepared.parentId; ordinal = prepared.next;
|
|
24
|
+
await checkpoint({ parent_id: parentId, prepared_children: ordinal, aggregate_success: false });
|
|
25
|
+
const status = await post('wave-status', { parentId });
|
|
26
|
+
if (!Number.isSafeInteger(status.census?.children) || status.census.children < ordinal) throw Error('Invalid complete census response.');
|
|
27
|
+
if (ordinal === status.census.children) break;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
if (!id(parentId)) throw Error('Invalid bounded parent identifier.');
|
|
31
|
+
const sealed = await post('seal-parent', { parentId });
|
|
32
|
+
if (!digest(sealed.fingerprint) || sealed.parentId !== parentId || !Array.isArray(sealed.children)) throw Error('Invalid sealed parent response.');
|
|
33
|
+
const approval = arrangement.parentApproval;
|
|
34
|
+
if (!approval) return { parentId, fingerprint: sealed.fingerprint, state: 'awaiting_parent_approval', aggregateSuccess: false, children: sealed.children };
|
|
35
|
+
if (approval.fingerprint !== sealed.fingerprint || !Array.isArray(arrangement.children) || arrangement.children.length !== sealed.children.length
|
|
36
|
+
|| arrangement.children.some((c, i) => c.ordinal !== i || c.fingerprint !== sealed.children[i].fingerprint)) throw Error('Business approvals differ from the immutable server children.');
|
|
37
|
+
const parentBinding = { parentFingerprint: approval.fingerprint, parentApprovalRecordId: approval.id, parentApprovedAt: approval.at };
|
|
38
|
+
await post('review-parent', { parentId }, parentBinding);
|
|
39
|
+
for (const child of arrangement.children) {
|
|
40
|
+
// Approvals are request/grant inputs to Business, never proof from this task.
|
|
41
|
+
if (!child.approval) return { parentId, state: 'awaiting_child_approval', ordinal: child.ordinal, aggregateSuccess: false };
|
|
42
|
+
for (;;) {
|
|
43
|
+
const status = await post('wave-status', { parentId });
|
|
44
|
+
if (status.state === 'withdrawn') throw Error('Bounded parent was withdrawn.');
|
|
45
|
+
const current = status.children?.[child.ordinal];
|
|
46
|
+
if (!current || current.fingerprint !== child.fingerprint) throw Error('Server child inventory changed.');
|
|
47
|
+
if (current.childSuccess) break;
|
|
48
|
+
const applied = await post('apply-child', { parentId, ordinal: child.ordinal, reviewedFingerprint: child.fingerprint }, { ...parentBinding, childFingerprint: child.fingerprint });
|
|
49
|
+
await checkpoint({ parent_id: parentId, child: child.ordinal, job_id: applied.job?.id, child_success: applied.childSuccess === true, aggregate_success: false });
|
|
50
|
+
if (!applied.job || ['failed','undone','rolled_back'].includes(applied.job.state)) return { parentId, state: 'partial', ordinal: child.ordinal, aggregateSuccess: false };
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
const aggregate = await post('finalize-parent', { parentId }, parentBinding);
|
|
54
|
+
if (aggregate.parentId !== parentId || aggregate.fingerprint !== sealed.fingerprint || aggregate.aggregateSuccess !== true || aggregate.completeHead !== parentId) throw Error('Destination did not return a complete aggregate receipt.');
|
|
55
|
+
await checkpoint({ parent_id: parentId, aggregate_success: true });
|
|
56
|
+
return aggregate;
|
|
57
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { sendBoundedWaves } from './bounded-waves.mjs';
|
|
2
|
+
import { ExportWaiting } from './vendor-door.mjs';
|
|
1
3
|
import { withUploadedBundle, door, checkedPreview, plural } from './vendor-door.mjs';
|
|
2
4
|
|
|
3
5
|
// Why the door refused a catch-up under the owner's standing approval
|
|
@@ -23,9 +25,14 @@ const beyond = {
|
|
|
23
25
|
export function runCatchup({ lease, task, grant, directory, say, ownerPage }) {
|
|
24
26
|
return withUploadedBundle({ lease, task, grant, directory, say,
|
|
25
27
|
doing: (name, origin, keyed) => `Bringing over recent ${name} changes through the shipped read-only adapter. ${keyed ? 'The key stays on this computer; records' : 'Records'} go only to the app's import door at ${origin}.` },
|
|
26
|
-
async ({ origin, uploadId, bundle, fresh }) => {
|
|
28
|
+
async ({ origin, uploadId, bundle, fresh, preview: uploadedPreview }) => {
|
|
29
|
+
if (task.bounded_waves) {
|
|
30
|
+
const result=await sendBoundedWaves({arrangement:{...task.bounded_waves,catchup:true},uploadId,fresh,request:door,signal:lease.signal,checkpoint:cp=>lease.checkpointSaved(cp)});
|
|
31
|
+
if (!result.aggregateSuccess) throw new ExportWaiting(`Bounded catch-up ${result.parentId} is ${result.state}; no complete migration receipt.`, 'configuration_required');
|
|
32
|
+
return {outputs:{parent_id:result.parentId,parent_fingerprint:result.fingerprint,aggregate_success:true},said:`Destination ${origin} reconciled the complete bounded catch-up ${result.parentId}.`};
|
|
33
|
+
}
|
|
27
34
|
let planGrant = await fresh('plan'), refused = null, preview;
|
|
28
|
-
try { preview = await door(planGrant, { uploadId, catchup: true }, lease.signal); }
|
|
35
|
+
try { preview = uploadedPreview ?? await door(planGrant, { uploadId, catchup: true }, lease.signal); }
|
|
29
36
|
catch (error) {
|
|
30
37
|
const reason = planGrant.catchup === 'bounded' && /^catchup_requires_proposal:([a-z0-9_]+)$/.exec(error.doorCode ?? '')?.[1];
|
|
31
38
|
if (!reason) throw error;
|
package/cli/business/command.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { setupCommand } from './setup-command.mjs';
|
|
2
2
|
import { scanCommand } from './scan-command.mjs';
|
|
3
3
|
import { connect } from './connect.mjs';
|
|
4
|
-
import { vendorKey, disconnect, status } from './local-commands.mjs';
|
|
4
|
+
import { vendorKey, legacyPackage, disconnect, status } from './local-commands.mjs';
|
|
5
5
|
import { next, answer, token } from './questions.mjs';
|
|
6
6
|
import { run } from './supervisor.mjs';
|
|
7
7
|
import { describe } from './errors.mjs';
|
|
@@ -36,12 +36,16 @@ Advanced commands (Usage: rer business VERB)
|
|
|
36
36
|
Catalog vendor or none; uses, sells, both, product-dependency
|
|
37
37
|
vendor-key add|remove VENDOR Keep a read-only vendor key in OS credential storage; the owner
|
|
38
38
|
types it in their own terminal, never in chat or a command
|
|
39
|
+
Zendesk: single-line {"client_id":"…","client_secret":"…"}; remove
|
|
40
|
+
revokes local export and deletes its retained snapshot
|
|
41
|
+
legacy-package add|remove|status DIR
|
|
42
|
+
Register a verified HappyToHelp legacy package (no key)
|
|
39
43
|
token MCP headersHelper output; send directly to the MCP client
|
|
40
44
|
status [--json] Pairing, service, current task and checkpoint
|
|
41
45
|
disconnect Revoke this computer, then remove local credentials and additions;
|
|
42
46
|
inside an agent sandbox it names the one command that finishes
|
|
43
|
-
Vendors: sentry, uptimerobot, pagerduty, calendly (with a read-only key); sorry (its public page,
|
|
44
|
-
no key). Owner approvals stay in the browser.
|
|
47
|
+
Vendors: sentry, uptimerobot, pagerduty, calendly, zendesk (with a read-only key); sorry (its public page,
|
|
48
|
+
no key); happytohelp-legacy (a local package, no key). Owner approvals stay in the browser.
|
|
45
49
|
State lives in $RER_BUSINESS_STATE_DIR when set, otherwise in ~/.rer/business.
|
|
46
50
|
In an agent sandbox that writes only inside its working folder, use RER_BUSINESS_STATE_DIR="$PWD/.rer-business".
|
|
47
51
|
`;
|
|
@@ -98,6 +102,7 @@ export async function businessCommand(argv) {
|
|
|
98
102
|
if (options.origin !== undefined && options.code === undefined && options.business === undefined) originUrl(options.origin);
|
|
99
103
|
await connect(options);
|
|
100
104
|
} else if (verb === 'vendor-key' && args.length === 2) await vendorKey(...args);
|
|
105
|
+
else if (verb === 'legacy-package' && args.length === 2 && ['add', 'remove', 'status'].includes(args[0])) await legacyPackage(...args);
|
|
101
106
|
else if (verb === 'answer' && [3, 4].includes(args.length)) await answer(...args);
|
|
102
107
|
else if (verb === 'run' && (!args.length || args.length === 1 && args[0] === '--once')) await run({ once: args[0] === '--once' });
|
|
103
108
|
else if (verb === 'status' && args.length === 1 && args[0] === '--json') await status({ json: true });
|
|
@@ -29,7 +29,17 @@ function since(window) {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
const shapes = {
|
|
32
|
+
zendesk(value) {
|
|
33
|
+
closed(value, ['account'], ['mode','highWater']);
|
|
34
|
+
const account = closed(value.account, ['subdomain']);
|
|
35
|
+
if (typeof account.subdomain !== 'string' || !/^(?!-)[a-z0-9-]{1,63}(?<!-)$/u.test(account.subdomain)
|
|
36
|
+
|| !['snapshot','sweep','census','retire'].includes(value.mode ?? 'snapshot')
|
|
37
|
+
|| value.highWater !== undefined && (!Number.isSafeInteger(value.highWater) || value.highWater < 0)
|
|
38
|
+
|| value.mode === 'sweep' && value.highWater === undefined) invalid();
|
|
39
|
+
return { account: { subdomain: account.subdomain }, mode: value.mode ?? 'snapshot', ...(value.highWater !== undefined ? { highWater: value.highWater } : {}) };
|
|
40
|
+
},
|
|
32
41
|
uptimerobot: () => ({}),
|
|
42
|
+
'happytohelp-legacy': () => ({}),
|
|
33
43
|
sentry(value, now) {
|
|
34
44
|
closed(value, ['organization', 'window']);
|
|
35
45
|
const organization = closed(value.organization, ['id', 'slug', 'region']);
|
|
@@ -57,13 +67,13 @@ const shapes = {
|
|
|
57
67
|
},
|
|
58
68
|
};
|
|
59
69
|
/** What each vendor's inputs are, in the owner's terms, while a task lacks them. */
|
|
60
|
-
export const inputNeeds = { sentry: 'its Sentry organization and export window', pagerduty: 'its export window', sorry: "its status page's address" };
|
|
70
|
+
export const inputNeeds = { zendesk: 'its Zendesk account subdomain', sentry: 'its Sentry organization and export window', pagerduty: 'its export window', sorry: "its status page's address" };
|
|
61
71
|
|
|
62
72
|
/** The shipped adapter's inputs for `vendor` from the leased task's
|
|
63
73
|
* `export_inputs`, or null when the task carries none yet. */
|
|
64
74
|
export function exportInputs(vendor, task, now = Date.now()) {
|
|
65
75
|
if (!Object.hasOwn(shapes, vendor)) return null;
|
|
66
|
-
if (vendor === 'uptimerobot') return {};
|
|
76
|
+
if (vendor === 'uptimerobot' || vendor === 'happytohelp-legacy') return {};
|
|
67
77
|
if (task.export_inputs === undefined || task.export_inputs === null) return null;
|
|
68
78
|
return shapes[vendor](task.export_inputs, now);
|
|
69
79
|
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { sendBoundedWaves } from './bounded-waves.mjs';
|
|
2
|
+
import { ExportWaiting } from './vendor-door.mjs';
|
|
1
3
|
import { withUploadedBundle, door, checkedPreview, plural } from './vendor-door.mjs';
|
|
2
4
|
|
|
3
5
|
/** Perform a claimed export.vendor: the shipped adapter's bundle uploaded into
|
|
@@ -7,8 +9,13 @@ import { withUploadedBundle, door, checkedPreview, plural } from './vendor-door.
|
|
|
7
9
|
export function runExport({ lease, task, grant, directory, say, ownerPage }) {
|
|
8
10
|
return withUploadedBundle({ lease, task, grant, directory, say,
|
|
9
11
|
doing: (name, origin, keyed) => `Exporting ${name} through the shipped read-only adapter. ${keyed ? 'The key stays on this computer; records' : 'Records'} go only to the app's import door at ${origin}.` },
|
|
10
|
-
async ({ origin, uploadId, bundle, fresh }) => {
|
|
11
|
-
|
|
12
|
+
async ({ origin, uploadId, bundle, fresh, preview: uploadedPreview }) => {
|
|
13
|
+
if (task.bounded_waves) {
|
|
14
|
+
const result=await sendBoundedWaves({arrangement:task.bounded_waves,uploadId,fresh,request:door,signal:lease.signal,checkpoint:cp=>lease.checkpointSaved(cp)});
|
|
15
|
+
if (!result.aggregateSuccess) throw new ExportWaiting(`Bounded import ${result.parentId} is ${result.state}; no complete migration receipt.`, 'configuration_required');
|
|
16
|
+
return {outputs:{parent_id:result.parentId,parent_fingerprint:result.fingerprint,aggregate_success:true},said:`Destination ${origin} reconciled the complete bounded import ${result.parentId}.`};
|
|
17
|
+
}
|
|
18
|
+
const preview = uploadedPreview ?? await door(await fresh('plan'), { uploadId, ...(bundle.mode === 'sweep' ? { catchup: true } : {}) }, lease.signal), { records, created, updated, unchanged, rejected } = checkedPreview(preview, bundle);
|
|
12
19
|
// Identifiers only: Business reads that plan from the door with its own grants
|
|
13
20
|
// and completes the task only if it is the plan of exactly this bundle.
|
|
14
21
|
return { outputs: { upload_id: uploadId, plan_id: preview.planId, bundle_sha256: bundle.sha256 },
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { constants } from 'node:fs';
|
|
2
|
+
import { open, readFile, realpath, mkdir, chmod, rename, unlink, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
5
|
+
import { assertChunk } from './outbound.mjs';
|
|
6
|
+
import { stateDirectory } from './state.mjs';
|
|
7
|
+
|
|
8
|
+
export const LEGACY_FORMAT = 'happytohelp-legacy-chunks-v1';
|
|
9
|
+
const MAX_BYTES = 32 * 1024 ** 3, MAX_JSON = 32 * 1024 ** 2, MAX_FILES = 100_000;
|
|
10
|
+
const sha = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
11
|
+
const count = n => Number.isSafeInteger(n) && n >= 0;
|
|
12
|
+
const registrationPath = () => join(stateDirectory(), 'legacy-package.json');
|
|
13
|
+
const refused = (file, row, reason) => { throw Error(`Legacy package refused: ${file}, row ${row}: ${reason}. Use HappyToHelp's repass, then register the package again.`); };
|
|
14
|
+
const filename = name => typeof name === 'string' && /^\w[\w.-]*$/.test(name) && name !== 'index.ndjson' && name !== 'rer-package.json';
|
|
15
|
+
|
|
16
|
+
/** Every opened source is a regular file, never a symlink; hashing is streamed. */
|
|
17
|
+
async function inspect(root, name, expected, json = false) {
|
|
18
|
+
if (!filename(name)) refused('manifest.json', 1, 'invalid filename');
|
|
19
|
+
let file;
|
|
20
|
+
try { file = await open(join(root, name), constants.O_RDONLY | constants.O_NOFOLLOW); }
|
|
21
|
+
catch { refused(name, 0, 'file unavailable'); }
|
|
22
|
+
try {
|
|
23
|
+
const info = await file.stat();
|
|
24
|
+
if (!info.isFile() || !count(expected.bytes) || info.size !== expected.bytes || info.size > (json ? MAX_JSON : MAX_BYTES)) refused(name, 0, 'size mismatch or limit');
|
|
25
|
+
const hash = createHash('sha256'), chunks = [];
|
|
26
|
+
for await (const chunk of file.createReadStream({ autoClose: false })) { hash.update(chunk); if (json) chunks.push(chunk); }
|
|
27
|
+
if (!/^[a-f0-9]{64}$/.test(expected.sha256 ?? '') || hash.digest('hex') !== expected.sha256) refused(name, 0, 'SHA-256 mismatch');
|
|
28
|
+
if (!json) return;
|
|
29
|
+
let value;
|
|
30
|
+
try { value = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks))); } catch { refused(name, 1, 'invalid JSON'); }
|
|
31
|
+
const rows = Array.isArray(value) ? value : [value];
|
|
32
|
+
// Check decoded strings as well as raw JSON, so escaping cannot hide a token.
|
|
33
|
+
const detect = value => {
|
|
34
|
+
if (typeof value === 'string') assertChunk(Buffer.from(value), []);
|
|
35
|
+
else if (value && typeof value === 'object') for (const [key, child] of Object.entries(value)) { assertChunk(Buffer.from(key), []); detect(child); }
|
|
36
|
+
};
|
|
37
|
+
for (let i = 0; i < rows.length; i++) {
|
|
38
|
+
try { assertChunk(Buffer.from(JSON.stringify(rows[i])), []); detect(rows[i]); } catch { refused(name, i + 1, 'outbound detector'); }
|
|
39
|
+
}
|
|
40
|
+
return value;
|
|
41
|
+
} finally { await file.close(); }
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Validate the complete legacy database package before any upload. Media are customer originals. */
|
|
45
|
+
export async function verifyLegacyPackage(directory) {
|
|
46
|
+
let path, handle, manifestBytes;
|
|
47
|
+
try { path = await realpath(resolve(directory)); handle = await open(join(path, 'manifest.json'), constants.O_RDONLY | constants.O_NOFOLLOW); }
|
|
48
|
+
catch { refused('manifest.json', 0, 'file unavailable'); }
|
|
49
|
+
try { const info = await handle.stat(); if (!info.isFile() || info.size > MAX_JSON) refused('manifest.json', 1, 'manifest size'); manifestBytes = await handle.readFile(); }
|
|
50
|
+
finally { await handle.close(); }
|
|
51
|
+
let manifest;
|
|
52
|
+
try { manifest = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(manifestBytes)); } catch { refused('manifest.json', 1, 'invalid JSON'); }
|
|
53
|
+
if (manifest?.format !== LEGACY_FORMAT || !Array.isArray(manifest.chunks) || !Array.isArray(manifest.context) || !Array.isArray(manifest.objects)
|
|
54
|
+
|| !manifest.source?.rowCounts || typeof manifest.source.rowCounts !== 'object' || Array.isArray(manifest.source.rowCounts)
|
|
55
|
+
|| !Array.isArray(manifest.source.attachments) || manifest.attachmentsComplete !== true) refused('manifest.json', 1, 'invalid legacy manifest');
|
|
56
|
+
await inspect(path, 'manifest.json', { bytes: manifestBytes.length, sha256: sha(manifestBytes) }, true);
|
|
57
|
+
const files = [{ path: 'manifest.json', file: join(path, 'manifest.json'), bytes: manifestBytes.length, sha256: sha(manifestBytes) }], seen = new Set(['manifest.json']), rowCounts = {};
|
|
58
|
+
let totalBytes = manifestBytes.length;
|
|
59
|
+
const add = (name, bytes, sha256) => {
|
|
60
|
+
if (!filename(name) || seen.has(name) || files.length >= MAX_FILES) refused('manifest.json', 1, 'duplicate filename or file limit');
|
|
61
|
+
if (!count(bytes) || (totalBytes += bytes) > MAX_BYTES) refused(name, 0, 'package byte limit');
|
|
62
|
+
seen.add(name); files.push({ path: name, file: join(path, name), bytes, sha256 });
|
|
63
|
+
};
|
|
64
|
+
for (const [name, n] of Object.entries(manifest.source.rowCounts)) if (!name || !count(n)) refused('manifest.json', 1, 'invalid row counts');
|
|
65
|
+
for (const [parts, primary] of [[manifest.chunks, true], [manifest.context, false]]) for (const part of parts) {
|
|
66
|
+
if (!part || typeof part.table !== 'string' || !Object.hasOwn(manifest.source.rowCounts, part.table) || !count(part.count)) refused('manifest.json', 1, 'invalid chunk count or table');
|
|
67
|
+
add(part.file, part.bytes, part.sha256);
|
|
68
|
+
const rows = await inspect(path, part.file, part, true);
|
|
69
|
+
if (!Array.isArray(rows) || rows.length !== part.count) refused(part.file, 0, 'row count mismatch');
|
|
70
|
+
if (primary) Object.defineProperty(rowCounts, part.table, { value: (Object.hasOwn(rowCounts, part.table) ? rowCounts[part.table] : 0) + rows.length, enumerable: true, configurable: true });
|
|
71
|
+
}
|
|
72
|
+
for (const [table, n] of Object.entries(manifest.source.rowCounts)) if ((Object.hasOwn(rowCounts, table) ? rowCounts[table] : 0) !== n) refused('manifest.json', 1, 'table count mismatch');
|
|
73
|
+
const keys = new Map(), media = new Set(); let objectBytes = 0;
|
|
74
|
+
for (const object of manifest.objects) {
|
|
75
|
+
if (!object || typeof object.key !== 'string' || !object.key.startsWith('legacy/') || object.key.split('/').includes('..') || object.mediaId == null
|
|
76
|
+
|| keys.has(object.key) || media.has(String(object.mediaId)) || !count(object.size)) refused('manifest.json', 1, 'invalid object identity');
|
|
77
|
+
keys.set(object.key, String(object.mediaId)); media.add(String(object.mediaId));
|
|
78
|
+
add(object.file, object.size, object.sha256);
|
|
79
|
+
await inspect(path, object.file, { bytes: object.size, sha256: object.sha256 });
|
|
80
|
+
objectBytes += object.size;
|
|
81
|
+
}
|
|
82
|
+
const attachments = manifest.source.attachments;
|
|
83
|
+
if (attachments.length !== manifest.objects.length || new Set(attachments.map(e => e?.key)).size !== attachments.length
|
|
84
|
+
|| attachments.some(e => !e || keys.get(e.key) !== String(e.mediaId))) refused('manifest.json', 1, 'attachment count or identity mismatch');
|
|
85
|
+
const bytes = files.reduce((n, f) => n + f.bytes, 0);
|
|
86
|
+
if (!count(bytes) || bytes > MAX_BYTES) refused('manifest.json', 1, 'package byte limit');
|
|
87
|
+
return { path, manifestSha256: sha(manifestBytes), bytes, rowCounts: { ...manifest.source.rowCounts }, objects: { count: manifest.objects.length, bytes: objectBytes }, files, manifest };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export async function loadLegacyRegistration() {
|
|
91
|
+
try { return JSON.parse(await readFile(registrationPath(), 'utf8')); } catch (error) { if (error.code === 'ENOENT') return null; throw error; }
|
|
92
|
+
}
|
|
93
|
+
export async function registeredLegacyPackage() {
|
|
94
|
+
const saved = await loadLegacyRegistration();
|
|
95
|
+
if (!saved) throw Error('No legacy package is registered. Run rer business legacy-package add DIR.');
|
|
96
|
+
let current;
|
|
97
|
+
try { current = await verifyLegacyPackage(saved.path); } catch { throw Error(`Legacy package changed or is unavailable: ${saved.path}. Use HappyToHelp's repass and register it again.`); }
|
|
98
|
+
if (JSON.stringify(summary(current)) !== JSON.stringify(saved)) throw Error(`Legacy package changed: ${saved.path}. Register the package again.`);
|
|
99
|
+
return current;
|
|
100
|
+
}
|
|
101
|
+
const summary = ({ path, manifestSha256, bytes, rowCounts, objects }) => ({ path, manifestSha256, bytes, rowCounts, objects });
|
|
102
|
+
export async function legacyPackage(action, directory) {
|
|
103
|
+
if (!['add', 'remove', 'status'].includes(action)) throw Error('Usage: rer business legacy-package add|remove|status DIR');
|
|
104
|
+
if (action === 'add') {
|
|
105
|
+
const verified = await verifyLegacyPackage(directory), target = registrationPath();
|
|
106
|
+
await mkdir(stateDirectory(), { recursive: true, mode: 0o700 }); await chmod(stateDirectory(), 0o700);
|
|
107
|
+
const temporary = target + '.' + randomUUID() + '.tmp';
|
|
108
|
+
try { await writeFile(temporary, JSON.stringify(summary(verified)), { mode: 0o600, flag: 'wx' }); await rename(temporary, target); }
|
|
109
|
+
finally { await unlink(temporary).catch(error => { if (error.code !== 'ENOENT') throw error; }); }
|
|
110
|
+
}
|
|
111
|
+
const saved = await loadLegacyRegistration();
|
|
112
|
+
if (saved && saved.path !== await realpath(resolve(directory))) throw Error('That directory is not the registered legacy package.');
|
|
113
|
+
if (action === 'remove') await unlink(registrationPath()).catch(error => { if (error.code !== 'ENOENT') throw error; });
|
|
114
|
+
const result = action === 'remove' ? null : saved;
|
|
115
|
+
// Registration never prints paths or customer rows; a changed task names its package locally.
|
|
116
|
+
process.stdout.write(JSON.stringify(result ? { bytes: result.bytes, rowCounts: result.rowCounts, objects: result.objects, manifestSha256: result.manifestSha256 } : { registered: false }) + '\n');
|
|
117
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { legacyPackage } from './legacy-package.mjs';
|
|
1
2
|
import { writeFile, unlink } from 'node:fs/promises';
|
|
2
3
|
import { join } from 'node:path';
|
|
3
4
|
import { loadState, loadPending, saveState, stateDirectory, removeStateDirectory, removeWorkFiles, deviceCredentialHandle, liveSupervisor } from './state.mjs';
|
|
@@ -11,6 +12,7 @@ import { KEYED_VENDORS } from './vendors/index.mjs';
|
|
|
11
12
|
import { vendorHandle } from './vendor-door.mjs';
|
|
12
13
|
import { rerCommand, businessCommand } from './invocation.mjs';
|
|
13
14
|
import { describe } from './errors.mjs';
|
|
15
|
+
import { clientCredential, removeZendeskStores } from './vendors/zendesk.mjs';
|
|
14
16
|
import { supportsTaskKind } from './supervisor.mjs';
|
|
15
17
|
|
|
16
18
|
export async function paired() {
|
|
@@ -44,6 +46,7 @@ export async function forgetLocally(state, { stopRun = true } = {}) {
|
|
|
44
46
|
for (const handle of store ? state.vendorHandles ?? [] : []) if (await attempt('vendorKeys', () => store.delete(handle))) vendorKeys.push(handle);
|
|
45
47
|
const refused = store ? await attempt('credential', () => store.delete(deviceCredentialHandle(state))) : undefined;
|
|
46
48
|
await removeCarries();
|
|
49
|
+
await removeZendeskStores(directory);
|
|
47
50
|
if (!leftovers.length) { await removeStateDirectory(); return []; }
|
|
48
51
|
const unique = [...new Set(leftovers)];
|
|
49
52
|
const cause = refused && (refused.sandboxRefused ? `macOS Keychain refused to remove it inside this agent sandbox (security exited with status ${refused.osStatus ?? 'unknown'}).` : describe(refused));
|
|
@@ -65,6 +68,7 @@ export async function revokedByServer(state) {
|
|
|
65
68
|
if (await permissionsAdded()) leftovers.push('permissions');
|
|
66
69
|
const saved = { ...state, revoked: { at: new Date().toISOString(), leftovers, ...(cause ? { cause } : {}) } };
|
|
67
70
|
await saveState(saved);
|
|
71
|
+
await removeZendeskStores(stateDirectory());
|
|
68
72
|
await removeWorkFiles(['state.json', 'permissions.json', 'service.json', 'run.lock', 'carried.json']);
|
|
69
73
|
return saved;
|
|
70
74
|
}
|
|
@@ -74,10 +78,15 @@ export async function vendorKey(action, vendor) {
|
|
|
74
78
|
const { state } = await paired(), store = createCredentialStore(), handle = vendorHandle(vendor);
|
|
75
79
|
if (action === 'add') {
|
|
76
80
|
let secret = await readSecret();
|
|
77
|
-
try { await store.set(handle, secret); } finally { secret = undefined; }
|
|
81
|
+
try { if (vendor === 'zendesk') clientCredential(secret); await store.set(handle, secret); } finally { secret = undefined; }
|
|
78
82
|
state.vendorHandles = [...new Set([...(state.vendorHandles ?? []), handle])];
|
|
79
83
|
} else {
|
|
84
|
+
if (vendor === 'zendesk') await stopSupervisor(stateDirectory());
|
|
80
85
|
await store.delete(handle);
|
|
86
|
+
if (vendor === 'zendesk') {
|
|
87
|
+
await removeZendeskStores(stateDirectory());
|
|
88
|
+
state.vendorRevocations = { ...state.vendorRevocations, zendesk: true };
|
|
89
|
+
}
|
|
81
90
|
state.vendorHandles = (state.vendorHandles ?? []).filter(item => item !== handle);
|
|
82
91
|
}
|
|
83
92
|
await saveState(state);
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { constants } from 'node:fs';
|
|
2
|
+
import { open, mkdir, chmod, readFile, rename, unlink, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
5
|
+
|
|
6
|
+
const CHUNK = 65_536, MAX_BYTES = 32 * 1024 ** 3;
|
|
7
|
+
const hash = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
8
|
+
const id = value => typeof value === 'string' && /^[0-9A-Za-z][0-9A-Za-z_-]{0,127}$/.test(value);
|
|
9
|
+
const pathValid = value => typeof value === 'string' && /^[A-Za-z0-9_][A-Za-z0-9_./-]*$/.test(value) && !value.split('/').some(p => !p || p === '.' || p === '..');
|
|
10
|
+
async function durableJson(path, value) {
|
|
11
|
+
const temporary = path + '.' + randomUUID() + '.tmp';
|
|
12
|
+
const file = await open(temporary, 'wx', 0o600);
|
|
13
|
+
try { await file.writeFile(JSON.stringify(value)); await file.sync(); } finally { await file.close(); }
|
|
14
|
+
try { await rename(temporary, path); } finally { await unlink(temporary).catch(error => { if (error.code !== 'ENOENT') throw error; }); }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** Concatenated index and exact source files; no JSON reserialization of the package files. */
|
|
18
|
+
export async function buildPackage({ files, source, mode = 'snapshot', records, totals, quota, directory, signal }) {
|
|
19
|
+
if (!Number.isSafeInteger(quota) || quota < CHUNK || quota > 2 * 1024 ** 2) throw Error('Unsupported package grant byte quota.');
|
|
20
|
+
if (!Array.isArray(files) || !files.length || files.length > 100_000) throw Error('Invalid package files.');
|
|
21
|
+
let offset = 0; const names = new Set();
|
|
22
|
+
const entries = files.map(file => {
|
|
23
|
+
if (!pathValid(file.path) || file.path === 'index.ndjson' || names.has(file.path) || !Number.isSafeInteger(file.bytes) || file.bytes < 0 || !/^[a-f0-9]{64}$/.test(file.sha256)) throw Error('Invalid package file.');
|
|
24
|
+
names.add(file.path);
|
|
25
|
+
const entry = { path: file.path, bytes: file.bytes, sha256: file.sha256, offset };
|
|
26
|
+
offset += file.bytes;
|
|
27
|
+
if (!Number.isSafeInteger(offset) || offset > MAX_BYTES) throw Error('Package byte limit.');
|
|
28
|
+
return entry;
|
|
29
|
+
});
|
|
30
|
+
const index = Buffer.from(entries.map(row => JSON.stringify(row) + '\n').join(''));
|
|
31
|
+
if (index.length > quota || index.length + offset > MAX_BYTES) throw Error('Package index or stream byte limit.');
|
|
32
|
+
await mkdir(directory, { recursive: true, mode: 0o700 }); await chmod(directory, 0o700);
|
|
33
|
+
const segments = [], streamHash = createHash('sha256'); let pending = Buffer.alloc(0), bytes = 0;
|
|
34
|
+
const save = async value => {
|
|
35
|
+
const ordinal = segments.length, path = join(directory, `segment-${ordinal}`), temporary = path + '.tmp';
|
|
36
|
+
const file = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW, 0o600);
|
|
37
|
+
try { await file.writeFile(value); await file.sync(); } finally { await file.close(); }
|
|
38
|
+
await rename(temporary, path); segments.push({ bytes: value.length, sha256: hash(value), file: path });
|
|
39
|
+
};
|
|
40
|
+
const feed = async value => {
|
|
41
|
+
signal?.throwIfAborted(); streamHash.update(value); bytes += value.length;
|
|
42
|
+
let position = 0;
|
|
43
|
+
while (position < value.length) {
|
|
44
|
+
const length = Math.min(quota - pending.length, value.length - position);
|
|
45
|
+
pending = Buffer.concat([pending, value.subarray(position, position + length)]); position += length;
|
|
46
|
+
if (pending.length === quota) { await save(pending); pending = Buffer.alloc(0); }
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
await feed(index);
|
|
50
|
+
for (const file of files) {
|
|
51
|
+
const handle = await open(file.file, constants.O_RDONLY | constants.O_NOFOLLOW), digest = createHash('sha256'); let size = 0;
|
|
52
|
+
try {
|
|
53
|
+
const info = await handle.stat(); if (!info.isFile() || info.size !== file.bytes) throw Error('Package source changed before upload.');
|
|
54
|
+
for await (const chunk of handle.createReadStream({ autoClose: false, highWaterMark: CHUNK })) {
|
|
55
|
+
size += chunk.length; if (size > file.bytes) throw Error('Package source changed before upload.');
|
|
56
|
+
digest.update(chunk); await feed(chunk);
|
|
57
|
+
}
|
|
58
|
+
if (size !== file.bytes || digest.digest('hex') !== file.sha256) throw Error('Package source changed before upload.');
|
|
59
|
+
} finally { await handle.close(); }
|
|
60
|
+
}
|
|
61
|
+
if (pending.length) await save(pending);
|
|
62
|
+
const manifest = { format: 'rer-door-package/1', source, mode, stream: { bytes, sha256: streamHash.digest('hex') },
|
|
63
|
+
segments: segments.map(({ bytes, sha256 }) => ({ bytes, sha256 })), index: { path: 'index.ndjson', bytes: index.length, sha256: hash(index) }, records, totals };
|
|
64
|
+
const manifestBytes = Buffer.from(JSON.stringify(manifest));
|
|
65
|
+
if (manifestBytes.length > quota) throw Error('Package manifest exceeds the grant byte quota.');
|
|
66
|
+
const manifestPath = join(directory, 'rer-package.json'); await writeFile(manifestPath, manifestBytes, { mode: 0o600 });
|
|
67
|
+
return { manifest, directory, segments, manifestFile: { bytes: manifestBytes.length, sha256: hash(manifestBytes), file: manifestPath } };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Persist upload ids before their first request. Inventory is the authority after an interrupted ACK. */
|
|
71
|
+
export async function transferPackage({ prepared, fresh, request, signal, checkpoint, catchup = false, planAction = 'plan', planInput = {} }) {
|
|
72
|
+
const { manifestFile, segments, directory } = prepared, journalPath = join(directory, 'journal.json');
|
|
73
|
+
let journal;
|
|
74
|
+
try { journal = JSON.parse(await readFile(journalPath, 'utf8')); } catch (error) { if (error.code !== 'ENOENT') throw error; }
|
|
75
|
+
if (!journal || journal.manifestSha256 !== manifestFile.sha256) journal = { manifestSha256: manifestFile.sha256, uploads: {}, supported: false };
|
|
76
|
+
if (!journal.uploads || typeof journal.uploads !== 'object') throw Error('Invalid package journal.');
|
|
77
|
+
const save = () => durableJson(journalPath, journal);
|
|
78
|
+
let done = 0;
|
|
79
|
+
async function upload(file, name, key) {
|
|
80
|
+
let uploadId = journal.uploads[key], inventory = null;
|
|
81
|
+
if (uploadId !== undefined && !id(uploadId)) throw Error('Invalid package journal upload id.');
|
|
82
|
+
if (uploadId) {
|
|
83
|
+
try { inventory = await request(await fresh('uploads'), { uploadId }, signal); }
|
|
84
|
+
catch (error) { if (error.doorStatus !== 404) throw error; uploadId = null; }
|
|
85
|
+
}
|
|
86
|
+
if (!uploadId) { uploadId = randomUUID(); journal.uploads[key] = uploadId; await save(); }
|
|
87
|
+
const handle = await open(file.file, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
88
|
+
try {
|
|
89
|
+
const totalChunks = Math.ceil(file.bytes / CHUNK), received = new Map();
|
|
90
|
+
if (inventory) {
|
|
91
|
+
if (inventory.uploadId !== uploadId || inventory.bundleSha256 !== file.sha256 || !Array.isArray(inventory.received)) throw Error('Invalid package inventory.');
|
|
92
|
+
for (const row of inventory.received) {
|
|
93
|
+
if (!Number.isSafeInteger(row.ordinal) || row.ordinal < 0 || row.ordinal >= totalChunks || received.has(row.ordinal)) throw Error('Invalid package chunk inventory.');
|
|
94
|
+
received.set(row.ordinal, row);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
for (let index = 0; index < totalChunks; index++) {
|
|
98
|
+
signal.throwIfAborted();
|
|
99
|
+
const bytes = Buffer.alloc(Math.min(CHUNK, file.bytes - index * CHUNK)), { bytesRead } = await handle.read(bytes, 0, bytes.length, index * CHUNK);
|
|
100
|
+
if (bytesRead !== bytes.length) throw Error('Package spool changed.');
|
|
101
|
+
const row = received.get(index);
|
|
102
|
+
if (row && (row.sha256 !== hash(bytes) || row.size !== bytes.length)) throw Error('Package inventory differs from local bytes.');
|
|
103
|
+
if (!row) {
|
|
104
|
+
const result = await request(await fresh('uploads'), { uploadId, name, bundleSha256: file.sha256, totalBytes: file.bytes, totalChunks, index, chunkSha256: hash(bytes), base64: bytes.toString('base64') }, signal);
|
|
105
|
+
if (result.uploadId !== uploadId || result.bundleSha256 !== file.sha256 || index === totalChunks - 1 && result.complete !== true) throw Error('Invalid package upload answer.');
|
|
106
|
+
}
|
|
107
|
+
done += bytes.length;
|
|
108
|
+
await checkpoint({ bytes_done: done });
|
|
109
|
+
}
|
|
110
|
+
} finally { await handle.close(); }
|
|
111
|
+
return uploadId;
|
|
112
|
+
}
|
|
113
|
+
const uploadId = await upload(manifestFile, 'rer-package.json', 'manifest');
|
|
114
|
+
let preview;
|
|
115
|
+
try { preview = await request(await fresh(planAction), { ...planInput, uploadId, ...(catchup ? { catchup: true } : {}) }, signal); }
|
|
116
|
+
catch (error) {
|
|
117
|
+
if (error.doorStatus !== 409 || error.doorCode !== 'package_segment_missing' || !Array.isArray(error.doorDetails?.missing) || !error.doorDetails.missing.length
|
|
118
|
+
|| error.doorDetails.missing.some(row => !segments.some(s => s.sha256 === row.sha256 && s.bytes === row.bytes))) {
|
|
119
|
+
throw Object.assign(Error('SEED_PACKAGE_UNSUPPORTED: the seed did not accept a manifest-first package probe. No segments were sent.'), { cause: error });
|
|
120
|
+
}
|
|
121
|
+
journal.supported = true; await save();
|
|
122
|
+
}
|
|
123
|
+
if (preview && !journal.supported && planAction==='prepare' && preview.protocol==='rer-bounded-waves/1' && typeof preview.parentId==='string') {journal.supported=true;await save();}
|
|
124
|
+
if (preview && !journal.supported) throw Error('SEED_PACKAGE_UNSUPPORTED: the seed did not request package segments. No segments were sent.');
|
|
125
|
+
if (!preview) for (let index = 0; index < segments.length; index++) await upload(segments[index], 'rer-segment', String(index));
|
|
126
|
+
return { uploadId, bundle: { size: manifestFile.bytes, sha256: manifestFile.sha256 }, ...(preview ? { preview } : {}) };
|
|
127
|
+
}
|
package/cli/business/state.mjs
CHANGED
|
@@ -86,13 +86,13 @@ export async function checkStateWritable({ codeExists = true } = {}) {
|
|
|
86
86
|
|
|
87
87
|
// Files this CLI writes in its state folder. A folder named by RER_BUSINESS_STATE_DIR
|
|
88
88
|
// may hold other things, so only these are removed from it.
|
|
89
|
-
const owned = /^(?:(?:state|pairing|progress|permissions|service|carried)\.json|(?:run|pairing)\.lock|disconnect|service(?:-error)?\.log|discovery-[A-Za-z0-9_.:-]+\.(?:json|tmp)|export-[0-9a-f]{32}\.bundle|.*[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.tmp(?:\.check)?)$/;
|
|
89
|
+
const owned = /^(?:(?:state|pairing|progress|permissions|service|carried|legacy-package)\.json|(?:run|pairing)\.lock|disconnect|service(?:-error)?\.log|discovery-[A-Za-z0-9_.:-]+\.(?:json|tmp)|export-[0-9a-f]{32}\.bundle|package-[0-9a-f]{32}|.*[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.tmp(?:\.check)?)$/;
|
|
90
90
|
/** After a revocation whose local cleanup was refused, keep only what a later
|
|
91
91
|
* disconnect needs to finish: the state and the receipts of what remains. */
|
|
92
92
|
export async function removeWorkFiles(keep = ['state.json', 'permissions.json', 'service.json']) {
|
|
93
93
|
let entries;
|
|
94
94
|
try { entries = await readdir(stateDirectory()); } catch (error) { if (error.code === 'ENOENT') return; throw error; }
|
|
95
|
-
for (const name of entries) if (owned.test(name) && !keep.includes(name)) await rm(join(stateDirectory(), name), { force: true });
|
|
95
|
+
for (const name of entries) if (owned.test(name) && !keep.includes(name)) await rm(join(stateDirectory(), name), { recursive: true, force: true });
|
|
96
96
|
}
|
|
97
97
|
|
|
98
98
|
export async function removeStateDirectory() {
|
|
@@ -105,7 +105,7 @@ export async function removeStateDirectory() {
|
|
|
105
105
|
}
|
|
106
106
|
let entries;
|
|
107
107
|
try { entries = await readdir(directory); } catch (error) { if (error.code === 'ENOENT') return; throw error; }
|
|
108
|
-
for (const name of entries) if (owned.test(name)) await rm(join(directory, name), { force: true });
|
|
108
|
+
for (const name of entries) if (owned.test(name)) await rm(join(directory, name), { recursive: true, force: true });
|
|
109
109
|
await rmdir(directory).catch(error => { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error.code)) throw error; });
|
|
110
110
|
}
|
|
111
111
|
|