@hearthlog/tooling 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/hearthlog.mjs +6 -4
- package/package.json +2 -2
- package/src/init/init.ts +1 -0
- package/src/tenants/lock-file.ts +4 -3
- package/src/tenants/module-lock.ts +11 -2
- package/src/tenants/tenant-config.ts +1 -1
- package/src/vite/config.ts +13 -1
- package/templates/tenant/dependabot.yml.tmpl +13 -0
- package/templates/tenant/github-workflows/deploy.yml.tmpl +4 -1
- package/src/deploy/copy-tenant.ts +0 -395
- package/src/deploy/split-tenant.ts +0 -295
- package/src/tenants/tenant-copy-plan.ts +0 -119
package/bin/hearthlog.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// The `hearthlog` command: runs the TypeScript tools through tsx. `build`, `init`, `export`, `check-module`, `artifact`, and
|
|
2
|
+
// The `hearthlog` command: runs the TypeScript tools through tsx. `build`, `init`, `export`, `check-module`, `lock`, `artifact`, and
|
|
3
3
|
// `gallery-public:build` start before a site has generated files (or write their own), so they run alone; the
|
|
4
4
|
// `apple:*` commands run the build and then the tools of `@hearthlog/native` (a dependency of the site, not of this
|
|
5
|
-
// package); `ensure`, `deploy`, `deploy-ledger
|
|
5
|
+
// package); `ensure`, `deploy`, and `deploy-ledger` run the build and then a deploy tool; every other
|
|
6
6
|
// command goes to the CLI.
|
|
7
7
|
import { spawnSync } from 'node:child_process';
|
|
8
8
|
import { createRequire } from 'node:module';
|
|
@@ -33,7 +33,7 @@ if (native) {
|
|
|
33
33
|
|
|
34
34
|
// The deploy tools read the site's generated files, so each starts with the build (which does nothing when they are
|
|
35
35
|
// current). They work from a tenant repository because they run from this package, not from the host's scripts.
|
|
36
|
-
const deployTools = { ensure: 'deploy/ensure-tenant.ts', deploy: 'deploy/deploy-tenant.ts', 'deploy-ledger': 'deploy/deploy-ledger.ts'
|
|
36
|
+
const deployTools = { ensure: 'deploy/ensure-tenant.ts', deploy: 'deploy/deploy-tenant.ts', 'deploy-ledger': 'deploy/deploy-ledger.ts' };
|
|
37
37
|
if (deployTools[command]) {
|
|
38
38
|
const site = rest[rest.indexOf('--site') + 1];
|
|
39
39
|
const env = { ...process.env, ...(rest.includes('--site') && site ? { VITE_STOR_TENANT: site } : {}) };
|
|
@@ -46,7 +46,8 @@ if (deployTools[command]) {
|
|
|
46
46
|
if (command === 'plan') {
|
|
47
47
|
const site = rest[rest.indexOf('--site') + 1];
|
|
48
48
|
const env = { ...process.env, ...(rest.includes('--site') && site ? { VITE_STOR_TENANT: site } : {}) };
|
|
49
|
-
|
|
49
|
+
// The build's messages go to stderr: `plan --json` writes only the plan to stdout.
|
|
50
|
+
const built = spawnSync(process.execPath, [tsx, source('generate/build-generated.ts'), '--quiet'], { stdio: ['inherit', 2, 'inherit'], env }).status ?? 1;
|
|
50
51
|
if (built !== 0) process.exit(built);
|
|
51
52
|
}
|
|
52
53
|
|
|
@@ -55,6 +56,7 @@ const entry = {
|
|
|
55
56
|
init: 'init/init.ts',
|
|
56
57
|
export: 'init/export-tenant.ts',
|
|
57
58
|
'check-module': 'init/check-module.ts',
|
|
59
|
+
lock: 'tenants/module-lock.ts',
|
|
58
60
|
'gallery-public:build': 'public-workers/build-gallery-public.ts',
|
|
59
61
|
artifact: 'deploy/artifact.ts',
|
|
60
62
|
}[command];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hearthlog/tooling",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"license": "AGPL-3.0-only",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"tsx": "^4.23.15",
|
|
31
|
-
"@hearthlog/contracts": "0.1.
|
|
31
|
+
"@hearthlog/contracts": "0.1.4"
|
|
32
32
|
},
|
|
33
33
|
"bin": {
|
|
34
34
|
"hearthlog": "./bin/hearthlog.mjs"
|
package/src/init/init.ts
CHANGED
|
@@ -100,6 +100,7 @@ export function renderSite(options: InitOptions): Array<[string, string]> {
|
|
|
100
100
|
["pnpm-workspace.yaml", template("pnpm-workspace.yaml.tmpl")],
|
|
101
101
|
["patches/sqlocal@0.18.0.patch", template("patches/sqlocal@0.18.0.patch")],
|
|
102
102
|
[".github/workflows/deploy.yml", renderWorkflow(id)],
|
|
103
|
+
[".github/dependabot.yml", template("dependabot.yml.tmpl")],
|
|
103
104
|
[".gitignore", template("gitignore.tmpl")],
|
|
104
105
|
["README.md", render(template("README.md.tmpl"), values)],
|
|
105
106
|
];
|
package/src/tenants/lock-file.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The lock file's content: which modules a site ships and the exact manifest each was composed from. It
|
|
2
|
+
* The lock file's content: which modules a site ships and the exact manifest each was composed from. It names no version:
|
|
3
|
+
* a release that keeps a module's manifest changes nothing here (the package manager's lockfile adopts it), and a release
|
|
4
|
+
* that moves the contract changes `manifestSha256`, so `hearthlog lock --check` fails until the lock is regenerated and the
|
|
5
|
+
* diff names the module. It
|
|
3
6
|
* imports nothing generated, so `hearthlog init` can write one before a site has any generated files.
|
|
4
7
|
*/
|
|
5
8
|
|
|
@@ -14,7 +17,6 @@ import { modulesForFeatures, type LoadedModule } from "../compose/compose-module
|
|
|
14
17
|
export interface ModuleLockEntry {
|
|
15
18
|
id: string;
|
|
16
19
|
package: string;
|
|
17
|
-
version: string;
|
|
18
20
|
/** The site's enabled features this module provides. */
|
|
19
21
|
features: string[];
|
|
20
22
|
/** SHA-256 of the module's `hearthlog.module.json`. */
|
|
@@ -32,7 +34,6 @@ export function lockFor(modules: readonly LoadedModule[], features: readonly str
|
|
|
32
34
|
const entries: ModuleLockEntry[] = modulesForFeatures(modules, features).map(({ directory, manifest }) => ({
|
|
33
35
|
id: manifest.id,
|
|
34
36
|
package: manifest.package,
|
|
35
|
-
version: manifest.version,
|
|
36
37
|
features: manifest.features.map((feature) => feature.id).filter((id) => features.includes(id)),
|
|
37
38
|
manifestSha256: createHash("sha256").update(readFileSync(join(directory, "hearthlog.module.json"))).digest("hex"),
|
|
38
39
|
}));
|
|
@@ -33,8 +33,17 @@ export async function buildModuleLock(site: Pick<ResolvedSite, "features">): Pro
|
|
|
33
33
|
export async function moduleLockProblems(site: Pick<ResolvedSite, "id" | "features">): Promise<string[]> {
|
|
34
34
|
const expected = renderModuleLock(await buildModuleLock(site));
|
|
35
35
|
const current = await readFile(moduleLockPath(site.id), "utf8").catch(() => null);
|
|
36
|
-
if (current === null) return [`tenants/${site.id}/modules.lock.json is missing; run pnpm lock:modules`];
|
|
37
|
-
|
|
36
|
+
if (current === null) return [`tenants/${site.id}/modules.lock.json is missing; run pnpm lock:modules (hearthlog lock)`];
|
|
37
|
+
// Older locks also recorded each module's version; it is not part of the contract.
|
|
38
|
+
const withoutVersions = (text: string) => {
|
|
39
|
+
try {
|
|
40
|
+
const lock = JSON.parse(text) as { modules?: Array<Record<string, unknown>> };
|
|
41
|
+
return renderModuleLock({ ...lock, modules: (lock.modules ?? []).map(({ version: _version, ...entry }) => entry) } as unknown as ModuleLock);
|
|
42
|
+
} catch {
|
|
43
|
+
return text;
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
return withoutVersions(current) === expected ? [] : [`tenants/${site.id}/modules.lock.json is stale; run pnpm lock:modules (hearthlog lock)`];
|
|
38
47
|
}
|
|
39
48
|
|
|
40
49
|
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
@@ -105,7 +105,7 @@ export interface InfrastructureDefinition {
|
|
|
105
105
|
ledgerAdminEmail: string;
|
|
106
106
|
/** Shared Cloudflare Access team origin pushed to every site Worker. */
|
|
107
107
|
accessTeamDomain: string;
|
|
108
|
-
/** The Cloudflare account that holds the shared stack
|
|
108
|
+
/** The Cloudflare account that holds the shared stack. */
|
|
109
109
|
accountId?: string;
|
|
110
110
|
/** Browser application and team-wide SSO lifetimes, in whole hours. */
|
|
111
111
|
accessSessionDuration?: string;
|
package/src/vite/config.ts
CHANGED
|
@@ -23,6 +23,16 @@ import {
|
|
|
23
23
|
|
|
24
24
|
import { defineConfig, type ProxyOptions, type UserConfig } from 'vite';
|
|
25
25
|
|
|
26
|
+
/** The `@hearthlog/*` packages the site depends on (the platform and its modules). */
|
|
27
|
+
function hearthlogDependencies(): string[] {
|
|
28
|
+
try {
|
|
29
|
+
const manifest = JSON.parse(readFileSync('package.json', 'utf8')) as { dependencies?: Record<string, string>; devDependencies?: Record<string, string> };
|
|
30
|
+
return Object.keys({ ...manifest.dependencies, ...manifest.devDependencies }).filter((name) => name.startsWith('@hearthlog/') && name !== '@hearthlog/generated');
|
|
31
|
+
} catch {
|
|
32
|
+
return [];
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
26
36
|
function installedVersion(name: string): string {
|
|
27
37
|
try {
|
|
28
38
|
return JSON.parse(readFileSync(join('node_modules', name, 'package.json'), 'utf8')).version;
|
|
@@ -293,7 +303,9 @@ return defineConfig(({ command, mode }): UserConfig => {
|
|
|
293
303
|
},
|
|
294
304
|
],
|
|
295
305
|
optimizeDeps: {
|
|
296
|
-
|
|
306
|
+
// The platform and the modules ship Svelte and TypeScript source; installed from the registry they sit in node_modules,
|
|
307
|
+
// where the dependency scan cannot resolve their relative imports. They are compiled with the site, not pre-bundled.
|
|
308
|
+
exclude: ['sqlocal', 'microlighter', ...hearthlogDependencies()]
|
|
297
309
|
},
|
|
298
310
|
build: {
|
|
299
311
|
// Bundle Microlighter's lazy grammars, which Vite skips in dependencies by default.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Releases of the platform and of the modules arrive as pull requests that change the lockfile (and nothing else, unless a
|
|
2
|
+
# module's contract moved: then `hearthlog lock` rewrites tenants/<id>/modules.lock.json and the diff names the module).
|
|
3
|
+
# The build job of this repository's workflow builds each one; merge it to deploy, with the approval of the environment.
|
|
4
|
+
version: 2
|
|
5
|
+
updates:
|
|
6
|
+
- package-ecosystem: npm
|
|
7
|
+
directory: "/"
|
|
8
|
+
schedule:
|
|
9
|
+
interval: weekly
|
|
10
|
+
groups:
|
|
11
|
+
hearthlog:
|
|
12
|
+
patterns:
|
|
13
|
+
- "@hearthlog/*"
|
|
@@ -46,7 +46,10 @@ jobs:
|
|
|
46
46
|
pnpm exec hearthlog gallery-public:build --site {{id}}
|
|
47
47
|
pnpm exec vite build
|
|
48
48
|
pnpm exec hearthlog artifact manifest --site {{id}}
|
|
49
|
-
# After the build: the plan
|
|
49
|
+
# After the build: the lock and the plan read the generated files the build writes. The lock fails when a module's
|
|
50
|
+
# contract changed since `tenants/{{id}}/modules.lock.json` was written: run `pnpm exec hearthlog lock` and commit it.
|
|
51
|
+
- name: Check the module lock
|
|
52
|
+
run: pnpm exec hearthlog lock --check
|
|
50
53
|
- name: Plan (read-only)
|
|
51
54
|
run: pnpm exec hearthlog plan --site {{id}}
|
|
52
55
|
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
@@ -1,395 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env npx tsx
|
|
2
|
-
/**
|
|
3
|
-
* Copy one tenant into a new tenant on its own stack, with new identities.
|
|
4
|
-
*
|
|
5
|
-
* The source is only read. Both tenants keep working and diverge afterwards;
|
|
6
|
-
* nothing is merged back. See docs/tenant-copy.md for the full runbook.
|
|
7
|
-
*
|
|
8
|
-
* events Export the source app through the Ledger administrator API (the
|
|
9
|
-
* source app's own token is neither needed nor rotated), restore it
|
|
10
|
-
* into the target Ledger as a new app named after the target site,
|
|
11
|
-
* with new log IDs, and verify. Event IDs, timestamps, authors,
|
|
12
|
-
* action grouping, fork lineage, and hidden logs are kept. Writes
|
|
13
|
-
* `mapping.json` (IDs only, no tokens).
|
|
14
|
-
* files Deploy a short-lived copy Worker bound to the source and target
|
|
15
|
-
* buckets, copy invoice files, Gallery files, and the source app's
|
|
16
|
-
* Gallery publications with app and log IDs rewritten from the
|
|
17
|
-
* mapping, compare every object, then delete the Worker.
|
|
18
|
-
*
|
|
19
|
-
* Usage:
|
|
20
|
-
* pnpm tenant:copy -- events --from alpha --to jst [--allow-skipped-forks]
|
|
21
|
-
* pnpm tenant:copy -- files --from alpha --to jst --run <directory>
|
|
22
|
-
*
|
|
23
|
-
* Environment: the administrator tokens of both Ledgers (LEDGER_ADMIN_TOKEN or
|
|
24
|
-
* .wrangler/tenants/.admin-token for the shared Ledger, LEDGER_ADMIN_TOKEN_<SITE>
|
|
25
|
-
* or .wrangler/tenants/<id>/.admin-token for a site's own Ledger), optional
|
|
26
|
-
* LEDGER_BASE_URL / LEDGER_BASE_URL_<SITE> overrides, and a wrangler login for
|
|
27
|
-
* `files`.
|
|
28
|
-
*/
|
|
29
|
-
|
|
30
|
-
import { spawn } from "node:child_process";
|
|
31
|
-
import { randomBytes } from "node:crypto";
|
|
32
|
-
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
33
|
-
import { join, relative, resolve } from "node:path";
|
|
34
|
-
import { fileURLToPath } from "node:url";
|
|
35
|
-
|
|
36
|
-
import { dedicatedLedgerBaseUrl, resolveDedicatedAdminToken } from "./dedicated-ledger.js";
|
|
37
|
-
import { runWrangler } from "./deploy-runner.js";
|
|
38
|
-
import { resolveLedgerAdminToken, resolveLedgerBaseUrl } from "./ledger-credential.js";
|
|
39
|
-
import {
|
|
40
|
-
infrastructureFor,
|
|
41
|
-
loadTenantConfig,
|
|
42
|
-
REPOSITORY_ROOT,
|
|
43
|
-
siteById,
|
|
44
|
-
type InfrastructureDefinition,
|
|
45
|
-
type ResolvedSite,
|
|
46
|
-
} from "../tenants/tenant-config.js";
|
|
47
|
-
import { planFileCopies, validateMapping, type CopyJob, type TenantCopyMapping } from "../tenants/tenant-copy-plan.js";
|
|
48
|
-
import { WRANGLER_BASE } from "../tenants/wrangler-base.js";
|
|
49
|
-
import { ledgerScript } from "./ledger-scripts.js";
|
|
50
|
-
|
|
51
|
-
const COPY_ROOT = join(REPOSITORY_ROOT, ".wrangler", "tenant-copies");
|
|
52
|
-
const COPY_WORKER_CONFIG = "wrangler.tenant-copy.jsonc";
|
|
53
|
-
|
|
54
|
-
interface Options {
|
|
55
|
-
command: "events" | "files" | "help";
|
|
56
|
-
from: string;
|
|
57
|
-
to: string;
|
|
58
|
-
run?: string;
|
|
59
|
-
allowSkippedForks: boolean;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
interface LedgerAccess {
|
|
63
|
-
baseUrl: string;
|
|
64
|
-
adminToken: string;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
interface AppSummary {
|
|
68
|
-
id: string;
|
|
69
|
-
name: string;
|
|
70
|
-
revokedAt: string | null;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
interface LogSummary {
|
|
74
|
-
id: string;
|
|
75
|
-
headVersion: number;
|
|
76
|
-
sourceLogId: string | null;
|
|
77
|
-
sourceVersion: number | null;
|
|
78
|
-
hiddenAt: string | null;
|
|
79
|
-
status: string;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
interface OwnedPage { owned: string[]; cursor: string | null }
|
|
83
|
-
interface CopyPage { copied: number; skipped: number; bytes: number; cursor: string | null }
|
|
84
|
-
interface ComparePage { checked: number; problems: string[]; cursor: string | null }
|
|
85
|
-
|
|
86
|
-
export interface CopyRecord {
|
|
87
|
-
source: { site: string; appId: string };
|
|
88
|
-
target: { site: string; appId: string };
|
|
89
|
-
copiedAt: string;
|
|
90
|
-
manifestSha256: string;
|
|
91
|
-
logs: Array<{ sourceLogId: string; targetLogId: string; headVersion: number; hidden: boolean }>;
|
|
92
|
-
skippedLogIds: string[];
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
export function parseOptions(argv: string[]): Options {
|
|
96
|
-
const args = argv.filter((argument) => argument !== "--");
|
|
97
|
-
const command = args[0];
|
|
98
|
-
const options: Options = {
|
|
99
|
-
command: command === "events" || command === "files" ? command : "help",
|
|
100
|
-
from: "",
|
|
101
|
-
to: "",
|
|
102
|
-
allowSkippedForks: false,
|
|
103
|
-
};
|
|
104
|
-
for (let index = 1; index < args.length; index += 1) {
|
|
105
|
-
const argument = args[index];
|
|
106
|
-
const value = () => {
|
|
107
|
-
const next = args[index + 1]?.trim();
|
|
108
|
-
if (!next) throw new Error(`${argument} requires a value`);
|
|
109
|
-
index += 1;
|
|
110
|
-
return next;
|
|
111
|
-
};
|
|
112
|
-
switch (argument) {
|
|
113
|
-
case "--from": options.from = value(); break;
|
|
114
|
-
case "--to": options.to = value(); break;
|
|
115
|
-
case "--run": options.run = value(); break;
|
|
116
|
-
case "--allow-skipped-forks": options.allowSkippedForks = true; break;
|
|
117
|
-
case "--help": case "-h": options.command = "help"; break;
|
|
118
|
-
default: throw new Error(`Unknown option: ${argument}`);
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
if (options.command !== "help" && (!options.from || !options.to)) throw new Error("--from and --to are required");
|
|
122
|
-
if (options.command === "files" && !options.run) throw new Error("files needs --run <directory> from the events step");
|
|
123
|
-
return options;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
async function ledgerAccess(site: ResolvedSite, shared: InfrastructureDefinition): Promise<LedgerAccess> {
|
|
127
|
-
if (site.deployment) {
|
|
128
|
-
const variable = site.id.toUpperCase().replace(/-/g, "_");
|
|
129
|
-
const adminToken = await resolveDedicatedAdminToken(site.id);
|
|
130
|
-
if (!adminToken) throw new Error(`No administrator token for the ${site.id} Ledger`);
|
|
131
|
-
return {
|
|
132
|
-
baseUrl: process.env[`LEDGER_BASE_URL_${variable}`]?.trim().replace(/\/$/, "") || dedicatedLedgerBaseUrl(site, shared),
|
|
133
|
-
adminToken,
|
|
134
|
-
};
|
|
135
|
-
}
|
|
136
|
-
const adminToken = await resolveLedgerAdminToken();
|
|
137
|
-
if (!adminToken) throw new Error("No administrator token for the shared Ledger (LEDGER_ADMIN_TOKEN)");
|
|
138
|
-
return { baseUrl: resolveLedgerBaseUrl(process.env.LEDGER_BASE_URL, shared.ledgerWorker, shared.workersDevDomain), adminToken };
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
async function adminApi<T>(ledger: LedgerAccess, route: string): Promise<T> {
|
|
142
|
-
const response = await fetch(`${ledger.baseUrl}${route}`, { headers: { Authorization: `Bearer ${ledger.adminToken}` } });
|
|
143
|
-
if (!response.ok) throw new Error(`GET ${route} failed with ${response.status}`);
|
|
144
|
-
return await response.json() as T;
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
async function activeApp(ledger: LedgerAccess, name: string): Promise<AppSummary | null> {
|
|
148
|
-
const { apps } = await adminApi<{ apps: AppSummary[] }>(ledger, "/admin/v1/apps");
|
|
149
|
-
const matches = apps.filter((app) => app.name === name && app.revokedAt === null);
|
|
150
|
-
if (matches.length > 1) throw new Error(`Ledger has ${matches.length} active apps named "${name}"`);
|
|
151
|
-
return matches[0] ?? null;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/** Run the logical backup script; stdout is its JSON result, never shown. */
|
|
155
|
-
function logicalBackup(args: string[], ledger: LedgerAccess): Promise<unknown> {
|
|
156
|
-
return new Promise((resolvePromise, reject) => {
|
|
157
|
-
const backup = ledgerScript("logical-backup", args);
|
|
158
|
-
const child = spawn(backup.command, backup.args, {
|
|
159
|
-
cwd: REPOSITORY_ROOT,
|
|
160
|
-
env: { ...process.env, LEDGER_BASE_URL: ledger.baseUrl, LEDGER_ADMIN_TOKEN: ledger.adminToken },
|
|
161
|
-
stdio: ["ignore", "pipe", "inherit"],
|
|
162
|
-
});
|
|
163
|
-
let output = "";
|
|
164
|
-
child.stdout.on("data", (chunk: Buffer) => { output += chunk.toString("utf8"); });
|
|
165
|
-
child.on("error", reject);
|
|
166
|
-
child.on("close", (code) => {
|
|
167
|
-
if (code !== 0) return reject(new Error(`logical backup ${args[0]} failed with exit code ${code}`));
|
|
168
|
-
try {
|
|
169
|
-
resolvePromise(JSON.parse(output));
|
|
170
|
-
} catch {
|
|
171
|
-
reject(new Error(`logical backup ${args[0]} printed no JSON result`));
|
|
172
|
-
}
|
|
173
|
-
});
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
async function copyEvents(options: Options): Promise<void> {
|
|
178
|
-
const { infrastructure, sites } = await loadTenantConfig();
|
|
179
|
-
const source = siteById(sites, options.from);
|
|
180
|
-
const target = siteById(sites, options.to);
|
|
181
|
-
if (!target.deployment) throw new Error(`Target ${target.id} must run on its own stack (tenants/${target.id}/deployment.json)`);
|
|
182
|
-
const sourceLedger = await ledgerAccess(source, infrastructure);
|
|
183
|
-
const targetLedger = await ledgerAccess(target, infrastructure);
|
|
184
|
-
if (sourceLedger.baseUrl === targetLedger.baseUrl) throw new Error("Source and target use the same Ledger");
|
|
185
|
-
|
|
186
|
-
const sourceApp = await activeApp(sourceLedger, source.ledgerAppName);
|
|
187
|
-
if (!sourceApp) throw new Error(`The source Ledger has no active app named "${source.ledgerAppName}"`);
|
|
188
|
-
const existing = await activeApp(targetLedger, target.ledgerAppName);
|
|
189
|
-
if (existing) {
|
|
190
|
-
throw new Error(
|
|
191
|
-
`The ${target.id} Ledger already has an app named "${target.ledgerAppName}" (${existing.id}). ` +
|
|
192
|
-
"A copy only goes into a new app; to redo it, revoke that app first (see docs/tenant-copy.md).",
|
|
193
|
-
);
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
const run = join(COPY_ROOT, `${target.id}-${new Date().toISOString().replace(/\D/g, "").slice(0, 14)}`);
|
|
197
|
-
await mkdir(run, { recursive: true, mode: 0o700 });
|
|
198
|
-
console.log(`Exporting ${source.id} (${sourceApp.id}) through the administrator API...`);
|
|
199
|
-
const exported = await logicalBackup(["export", "--app", sourceApp.id, "--output", join(run, "export")], sourceLedger) as {
|
|
200
|
-
manifestSha256: string;
|
|
201
|
-
skippedLogIds: string[];
|
|
202
|
-
};
|
|
203
|
-
if (exported.skippedLogIds.length > 0 && !options.allowSkippedForks) {
|
|
204
|
-
throw new Error(
|
|
205
|
-
`Forks still being prepared were skipped: ${exported.skippedLogIds.join(", ")}. ` +
|
|
206
|
-
"Let them finish and run again, or pass --allow-skipped-forks to copy without them.",
|
|
207
|
-
);
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
console.log(`Restoring into the ${target.id} Ledger as "${target.ledgerAppName}"...`);
|
|
211
|
-
const restored = await logicalBackup(["restore", "--input", join(run, "export"), "--name", target.ledgerAppName], targetLedger) as {
|
|
212
|
-
app: { id: string };
|
|
213
|
-
manifestSha256: string;
|
|
214
|
-
logs: Array<{ sourceLogId: string; targetLogId: string }>;
|
|
215
|
-
};
|
|
216
|
-
if (restored.manifestSha256 !== exported.manifestSha256) throw new Error("Restore read a different export");
|
|
217
|
-
|
|
218
|
-
const record = await verifyEvents(sourceLedger, targetLedger, sourceApp.id, restored.app.id, restored.logs);
|
|
219
|
-
const result: CopyRecord = {
|
|
220
|
-
source: { site: source.id, appId: sourceApp.id },
|
|
221
|
-
target: { site: target.id, appId: restored.app.id },
|
|
222
|
-
copiedAt: new Date().toISOString(),
|
|
223
|
-
manifestSha256: exported.manifestSha256,
|
|
224
|
-
logs: record,
|
|
225
|
-
skippedLogIds: exported.skippedLogIds,
|
|
226
|
-
};
|
|
227
|
-
await writeFile(join(run, "mapping.json"), `${JSON.stringify(result, null, 2)}\n`, { mode: 0o600 });
|
|
228
|
-
console.log(
|
|
229
|
-
`\nCopied ${record.length} log(s) into app ${restored.app.id}. Mapping: ${relative(REPOSITORY_ROOT, run)}/mapping.json\n` +
|
|
230
|
-
`Next: pnpm tenant:copy -- files --from ${source.id} --to ${target.id} --run ${relative(REPOSITORY_ROOT, run)}\n` +
|
|
231
|
-
`Then give the ${target.id} gateway a token: pnpm ensure:tenant -- --site ${target.id} --rotate-token`,
|
|
232
|
-
);
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
/** Heads, lineage, and hidden state match across the mapping; restore already compared every event page. */
|
|
236
|
-
async function verifyEvents(
|
|
237
|
-
sourceLedger: LedgerAccess,
|
|
238
|
-
targetLedger: LedgerAccess,
|
|
239
|
-
sourceAppId: string,
|
|
240
|
-
targetAppId: string,
|
|
241
|
-
pairs: ReadonlyArray<{ sourceLogId: string; targetLogId: string }>,
|
|
242
|
-
): Promise<CopyRecord["logs"]> {
|
|
243
|
-
const sourceLogs = new Map((await adminApi<{ logs: LogSummary[] }>(sourceLedger, `/admin/v1/apps/${sourceAppId}/logs`)).logs.map((log) => [log.id, log]));
|
|
244
|
-
const targetLogs = new Map((await adminApi<{ logs: LogSummary[] }>(targetLedger, `/admin/v1/apps/${targetAppId}/logs`)).logs.map((log) => [log.id, log]));
|
|
245
|
-
const mapped = new Map(pairs.map((pair) => [pair.sourceLogId, pair.targetLogId]));
|
|
246
|
-
if (targetLogs.size !== pairs.length) throw new Error(`Target app has ${targetLogs.size} logs, expected ${pairs.length}`);
|
|
247
|
-
return pairs.map(({ sourceLogId, targetLogId }) => {
|
|
248
|
-
const from = sourceLogs.get(sourceLogId);
|
|
249
|
-
const to = targetLogs.get(targetLogId);
|
|
250
|
-
if (!from || !to) throw new Error(`Log ${sourceLogId} is missing after the copy`);
|
|
251
|
-
// The source may have moved on since the export; the copy holds the exported head.
|
|
252
|
-
if (to.headVersion > from.headVersion) throw new Error(`Log ${targetLogId} is ahead of its source`);
|
|
253
|
-
const expectedParent = from.sourceLogId ? mapped.get(from.sourceLogId) ?? null : null;
|
|
254
|
-
if ((to.sourceLogId ?? null) !== expectedParent || (to.sourceVersion ?? null) !== (from.sourceVersion ?? null)) {
|
|
255
|
-
throw new Error(`Fork lineage of ${targetLogId} does not match ${sourceLogId}`);
|
|
256
|
-
}
|
|
257
|
-
if (Boolean(to.hiddenAt) !== Boolean(from.hiddenAt)) throw new Error(`Hidden state of ${targetLogId} does not match`);
|
|
258
|
-
return { sourceLogId, targetLogId, headVersion: to.headVersion, hidden: Boolean(to.hiddenAt) };
|
|
259
|
-
});
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
export function buildCopyWorkerConfig(
|
|
263
|
-
name: string,
|
|
264
|
-
source: InfrastructureDefinition,
|
|
265
|
-
target: InfrastructureDefinition,
|
|
266
|
-
compatibilityDate: string,
|
|
267
|
-
): Record<string, unknown> {
|
|
268
|
-
return {
|
|
269
|
-
name,
|
|
270
|
-
main: "tenant-copy/index.ts",
|
|
271
|
-
compatibility_date: compatibilityDate,
|
|
272
|
-
workers_dev: true,
|
|
273
|
-
preview_urls: false,
|
|
274
|
-
r2_buckets: [
|
|
275
|
-
{ binding: "SOURCE_INVOICE_PDFS", bucket_name: source.invoicePdfsBucket },
|
|
276
|
-
{ binding: "TARGET_INVOICE_PDFS", bucket_name: target.invoicePdfsBucket },
|
|
277
|
-
{ binding: "SOURCE_GALLERY_FILES", bucket_name: source.galleryFilesBucket },
|
|
278
|
-
{ binding: "TARGET_GALLERY_FILES", bucket_name: target.galleryFilesBucket },
|
|
279
|
-
{ binding: "SOURCE_GALLERY_PUBLIC", bucket_name: source.galleryPublicBucket },
|
|
280
|
-
{ binding: "TARGET_GALLERY_PUBLIC", bucket_name: target.galleryPublicBucket },
|
|
281
|
-
],
|
|
282
|
-
};
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
async function copyFiles(options: Options): Promise<void> {
|
|
286
|
-
const { infrastructure, sites } = await loadTenantConfig();
|
|
287
|
-
const source = siteById(sites, options.from);
|
|
288
|
-
const target = siteById(sites, options.to);
|
|
289
|
-
const run = resolve(REPOSITORY_ROOT, options.run!);
|
|
290
|
-
const record = JSON.parse(await readFile(join(run, "mapping.json"), "utf8")) as CopyRecord;
|
|
291
|
-
if (record.source.site !== source.id || record.target.site !== target.id) throw new Error("The mapping belongs to another copy");
|
|
292
|
-
const mapping: TenantCopyMapping = {
|
|
293
|
-
sourceAppId: record.source.appId,
|
|
294
|
-
targetAppId: record.target.appId,
|
|
295
|
-
logs: Object.fromEntries(record.logs.map((log) => [log.sourceLogId, log.targetLogId])),
|
|
296
|
-
};
|
|
297
|
-
validateMapping(mapping);
|
|
298
|
-
|
|
299
|
-
const name = `${target.id}-tenant-copy`;
|
|
300
|
-
const config = buildCopyWorkerConfig(name, infrastructureFor(source, infrastructure), infrastructureFor(target, infrastructure), WRANGLER_BASE.compatibility_date);
|
|
301
|
-
await writeFile(join(REPOSITORY_ROOT, COPY_WORKER_CONFIG), `${JSON.stringify(config, null, 2)}\n`, "utf8");
|
|
302
|
-
const token = randomBytes(32).toString("base64url");
|
|
303
|
-
const url = `https://${name}.${infrastructureFor(target, infrastructure).workersDevDomain}`;
|
|
304
|
-
|
|
305
|
-
console.log(`Deploying the short-lived copy Worker ${name}...`);
|
|
306
|
-
await runWrangler(COPY_WORKER_CONFIG, null, ["deploy"]);
|
|
307
|
-
try {
|
|
308
|
-
await runWrangler(COPY_WORKER_CONFIG, null, ["secret", "bulk"], { input: `${JSON.stringify({ COPY_TOKEN: token })}\n` });
|
|
309
|
-
const report = await copyFilesThrough(workerClient(url, token), mapping);
|
|
310
|
-
await writeFile(join(run, "files-report.json"), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 });
|
|
311
|
-
const total = report.jobs.reduce((sum, job) => sum + job.checked, 0);
|
|
312
|
-
console.log(`\nCopied and compared ${total} object(s), ${report.publications} publication(s). Report: ${relative(REPOSITORY_ROOT, run)}/files-report.json`);
|
|
313
|
-
} finally {
|
|
314
|
-
console.log(`Deleting the copy Worker ${name}...`);
|
|
315
|
-
await runWrangler(COPY_WORKER_CONFIG, null, ["delete", "--force"]);
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
export type CopyCall = <T>(path: string, body: unknown) => Promise<T>;
|
|
320
|
-
|
|
321
|
-
export interface FilesReport {
|
|
322
|
-
copiedAt: string;
|
|
323
|
-
publications: number;
|
|
324
|
-
jobs: Array<CopyJob & { copied: number; skipped: number; bytes: number; checked: number }>;
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
/** Copy, then compare, every planned prefix through a running copy Worker. */
|
|
328
|
-
export async function copyFilesThrough(call: CopyCall, mapping: TenantCopyMapping): Promise<FilesReport> {
|
|
329
|
-
const owned: string[] = [];
|
|
330
|
-
for (let cursor: string | null | undefined = undefined; cursor !== null;) {
|
|
331
|
-
const page: OwnedPage = await call<OwnedPage>("/publications", { appId: mapping.sourceAppId, cursor });
|
|
332
|
-
owned.push(...page.owned);
|
|
333
|
-
cursor = page.cursor;
|
|
334
|
-
}
|
|
335
|
-
const report: FilesReport = { copiedAt: new Date().toISOString(), publications: owned.length, jobs: [] };
|
|
336
|
-
for (const job of planFileCopies(mapping, owned)) {
|
|
337
|
-
const body = { ...job, ...(job.rewriteOwner ? { mapping } : {}) };
|
|
338
|
-
let copied = 0, skipped = 0, bytes = 0, checked = 0;
|
|
339
|
-
for (let cursor: string | null | undefined = undefined; cursor !== null;) {
|
|
340
|
-
const page: CopyPage = await call<CopyPage>("/copy", { ...body, cursor });
|
|
341
|
-
copied += page.copied; skipped += page.skipped; bytes += page.bytes; cursor = page.cursor;
|
|
342
|
-
}
|
|
343
|
-
for (let cursor: string | null | undefined = undefined; cursor !== null;) {
|
|
344
|
-
const page: ComparePage = await call<ComparePage>("/compare", { ...body, cursor });
|
|
345
|
-
if (page.problems.length > 0) throw new Error(`Copied objects differ under ${job.toPrefix}: ${page.problems.slice(0, 5).join(", ")}`);
|
|
346
|
-
checked += page.checked; cursor = page.cursor;
|
|
347
|
-
}
|
|
348
|
-
report.jobs.push({ ...job, copied, skipped, bytes, checked });
|
|
349
|
-
if (checked > 0) console.log(`${job.pair} ${job.fromPrefix} -> ${job.toPrefix}: ${checked} object(s), ${copied} copied`);
|
|
350
|
-
}
|
|
351
|
-
return report;
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
export function workerClient(url: string, token: string): CopyCall {
|
|
355
|
-
return async function call<T>(path: string, body: unknown): Promise<T> {
|
|
356
|
-
// A new workers.dev name can take a few seconds to answer.
|
|
357
|
-
for (let attempt = 0; ; attempt += 1) {
|
|
358
|
-
const response = await fetch(`${url}${path}`, {
|
|
359
|
-
method: "POST",
|
|
360
|
-
headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
|
|
361
|
-
body: JSON.stringify(body),
|
|
362
|
-
});
|
|
363
|
-
if (response.ok) return await response.json() as T;
|
|
364
|
-
if (attempt < 10 && (response.status === 404 || response.status >= 520)) {
|
|
365
|
-
await new Promise((wait) => setTimeout(wait, 3000));
|
|
366
|
-
continue;
|
|
367
|
-
}
|
|
368
|
-
const detail = await response.text();
|
|
369
|
-
throw new Error(`Copy Worker ${path} failed with ${response.status}: ${detail.slice(0, 300)}`);
|
|
370
|
-
}
|
|
371
|
-
};
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
function usage(): string {
|
|
375
|
-
return `Copy a tenant into a new tenant on its own stack (new identities; the source is only read).
|
|
376
|
-
|
|
377
|
-
Usage:
|
|
378
|
-
pnpm tenant:copy -- events --from <site> --to <site> [--allow-skipped-forks]
|
|
379
|
-
pnpm tenant:copy -- files --from <site> --to <site> --run <directory>`;
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
export async function main(argv = process.argv.slice(2)): Promise<void> {
|
|
383
|
-
const options = parseOptions(argv);
|
|
384
|
-
process.chdir(REPOSITORY_ROOT);
|
|
385
|
-
if (options.command === "events") await copyEvents(options);
|
|
386
|
-
else if (options.command === "files") await copyFiles(options);
|
|
387
|
-
else console.log(usage());
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
391
|
-
main().catch((error: unknown) => {
|
|
392
|
-
console.error(`\n${error instanceof Error ? error.message : String(error)}`);
|
|
393
|
-
process.exitCode = 1;
|
|
394
|
-
});
|
|
395
|
-
}
|
|
@@ -1,295 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env npx tsx
|
|
2
|
-
/**
|
|
3
|
-
* Splits a tenant of the shared stack onto its own Ledger and buckets, keeping
|
|
4
|
-
* every identity (docs/tenant-split-runbook.md). Each step changes nothing it
|
|
5
|
-
* cannot repeat, and none of them switches traffic except `cutover`, which you
|
|
6
|
-
* run yourself once `verify` passes.
|
|
7
|
-
*
|
|
8
|
-
* pnpm tenant:split -- declare --site bravo # write tenants/bravo/deployment.json (keep it local; see the runbook)
|
|
9
|
-
* pnpm tenant:split -- fence --site bravo --mode tenant_migrating|moved|off
|
|
10
|
-
* pnpm tenant:split -- transfer --site bravo # history: source Ledger -> own Ledger
|
|
11
|
-
* pnpm tenant:split -- files --site bravo # objects: shared buckets -> own buckets
|
|
12
|
-
* pnpm tenant:split -- verify --site bravo # checksums, file report, denied access
|
|
13
|
-
* pnpm tenant:split -- cutover --site bravo # verify, mint the gateway token, deploy the gateway (--skip-access: leave Access alone)
|
|
14
|
-
*
|
|
15
|
-
* The site must already have `tenants/<id>/deployment.json` and its stack must
|
|
16
|
-
* be provisioned empty (`pnpm hearthlog apply --site <id> --yes --stop-before ledger-app`: the import needs a
|
|
17
|
-
* Ledger with no application, and creates the application itself under its original id).
|
|
18
|
-
*
|
|
19
|
-
* `verify --skip-files` skips the file copy check and is accepted only against a Ledger on localhost (a rehearsal).
|
|
20
|
-
*/
|
|
21
|
-
|
|
22
|
-
import { spawn } from "node:child_process";
|
|
23
|
-
import { randomBytes } from "node:crypto";
|
|
24
|
-
import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
|
|
25
|
-
import { join, relative, resolve } from "node:path";
|
|
26
|
-
import { fileURLToPath } from "node:url";
|
|
27
|
-
|
|
28
|
-
import { identityMapping, validateMapping } from "../tenants/tenant-copy-plan.js";
|
|
29
|
-
import { moveApplication, sameChecksums, type AdminCall, type LogChecksum } from "@hearthlog/tooling";
|
|
30
|
-
|
|
31
|
-
import { buildCopyWorkerConfig, copyFilesThrough, workerClient, type FilesReport } from "./copy-tenant.js";
|
|
32
|
-
import { dedicatedLedgerBaseUrl, resolveDedicatedAdminToken } from "./dedicated-ledger.js";
|
|
33
|
-
import { runWrangler } from "./deploy-runner.js";
|
|
34
|
-
import { resolveLedgerAdminToken, resolveLedgerBaseUrl } from "./ledger-credential.js";
|
|
35
|
-
import {
|
|
36
|
-
gallerySweeperId,
|
|
37
|
-
infrastructureFor,
|
|
38
|
-
loadTenantConfig,
|
|
39
|
-
REPOSITORY_ROOT,
|
|
40
|
-
siteById,
|
|
41
|
-
type InfrastructureDefinition,
|
|
42
|
-
type ResolvedSite,
|
|
43
|
-
} from "../tenants/tenant-config.js";
|
|
44
|
-
import { WRANGLER_BASE } from "../tenants/wrangler-base.js";
|
|
45
|
-
|
|
46
|
-
const COPY_WORKER_CONFIG = "wrangler.tenant-move.jsonc";
|
|
47
|
-
|
|
48
|
-
export function movesDirectory(siteId: string): string {
|
|
49
|
-
return join(REPOSITORY_ROOT, ".wrangler", "tenant-moves", siteId);
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export function adminCaller(baseUrl: string, token: string): AdminCall {
|
|
53
|
-
return (method, path, body) =>
|
|
54
|
-
fetch(`${baseUrl.replace(/\/$/, "")}${path}`, {
|
|
55
|
-
method,
|
|
56
|
-
headers: { Authorization: `Bearer ${token}`, ...(body === undefined ? {} : { "Content-Type": "application/json" }) },
|
|
57
|
-
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
58
|
-
});
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
interface Stacks {
|
|
62
|
-
site: ResolvedSite;
|
|
63
|
-
shared: InfrastructureDefinition;
|
|
64
|
-
source: AdminCall;
|
|
65
|
-
target: () => Promise<AdminCall>;
|
|
66
|
-
sourceAppId: string;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
async function stacks(siteId: string): Promise<Stacks> {
|
|
70
|
-
const { infrastructure, sites } = await loadTenantConfig();
|
|
71
|
-
const site = siteById(sites, siteId);
|
|
72
|
-
if (!site.deployment) throw new Error(`${siteId} has no tenants/${siteId}/deployment.json: add it, then provision the stack with "pnpm hearthlog apply --site ${siteId} --yes --stop-before ledger-app"`);
|
|
73
|
-
const sourceToken = await resolveLedgerAdminToken();
|
|
74
|
-
if (!sourceToken) throw new Error("No administrator token for the shared Ledger (LEDGER_ADMIN_TOKEN)");
|
|
75
|
-
const source = adminCaller(resolveLedgerBaseUrl(process.env.LEDGER_BASE_URL, infrastructure.ledgerWorker, infrastructure.workersDevDomain), sourceToken);
|
|
76
|
-
const response = await source("GET", "/admin/v1/apps");
|
|
77
|
-
if (!response.ok) throw new Error(`Listing the shared Ledger's applications failed with ${response.status}`);
|
|
78
|
-
const apps = ((await response.json()) as { apps: Array<{ id: string; name: string; revokedAt: string | null }> }).apps;
|
|
79
|
-
const app = apps.find((candidate) => candidate.name === site.ledgerAppName && candidate.revokedAt === null);
|
|
80
|
-
if (!app) throw new Error(`The shared Ledger has no active application "${site.ledgerAppName}"`);
|
|
81
|
-
return {
|
|
82
|
-
site,
|
|
83
|
-
shared: infrastructure,
|
|
84
|
-
source,
|
|
85
|
-
sourceAppId: app.id,
|
|
86
|
-
target: async () => {
|
|
87
|
-
const token = await resolveDedicatedAdminToken(site.id);
|
|
88
|
-
if (!token) throw new Error(`No administrator token for the ${site.id} Ledger: provision it first`);
|
|
89
|
-
return adminCaller(process.env[`LEDGER_BASE_URL_${site.id.toUpperCase().replace(/-/g, "_")}`]?.trim() || dedicatedLedgerBaseUrl(site, infrastructure), token);
|
|
90
|
-
},
|
|
91
|
-
};
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
async function newRun(siteId: string): Promise<string> {
|
|
95
|
-
const directory = join(movesDirectory(siteId), new Date().toISOString().replace(/[:.]/g, "-"));
|
|
96
|
-
await mkdir(directory, { recursive: true });
|
|
97
|
-
return directory;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/** The latest run directory holding `file`, so `files` and `verify` follow `transfer`. */
|
|
101
|
-
async function latest(siteId: string, file: string): Promise<string | null> {
|
|
102
|
-
const runs = (await readdir(movesDirectory(siteId)).catch(() => [] as string[])).sort().reverse();
|
|
103
|
-
for (const run of runs) {
|
|
104
|
-
try {
|
|
105
|
-
await readFile(join(movesDirectory(siteId), run, file));
|
|
106
|
-
return join(movesDirectory(siteId), run);
|
|
107
|
-
} catch {
|
|
108
|
-
// Try the next older run.
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
return null;
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
async function fence(siteId: string, mode: string): Promise<void> {
|
|
115
|
-
if (!["tenant_migrating", "moved", "off"].includes(mode)) throw new Error('--mode must be tenant_migrating, moved, or off');
|
|
116
|
-
const { site, source, sourceAppId } = await stacks(siteId);
|
|
117
|
-
const body = mode === "moved" ? { mode, destination: new URL(site.hostname).origin } : { mode };
|
|
118
|
-
const response = await source("PUT", `/admin/v1/apps/${sourceAppId}/fence`, body);
|
|
119
|
-
if (!response.ok) throw new Error(`Setting the fence failed with ${response.status}: ${await response.text()}`);
|
|
120
|
-
console.log(`Fence on ${site.ledgerAppName} (${sourceAppId}): ${mode}`);
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
async function transfer(siteId: string, allowUnfenced: boolean): Promise<void> {
|
|
124
|
-
const { site, source, target, sourceAppId } = await stacks(siteId);
|
|
125
|
-
const run = await newRun(siteId);
|
|
126
|
-
const report = await moveApplication({ appId: sourceAppId, source, target: await target(), allowUnfenced, log: (message) => console.log(message) });
|
|
127
|
-
await writeFile(join(run, "transfer.json"), `${JSON.stringify({ site: site.id, ...report }, null, 2)}\n`, { mode: 0o600 });
|
|
128
|
-
console.log(`\nMoved ${report.logs} log(s), ${report.events} event(s), ${report.requests} idempotency record(s). Report: ${relative(REPOSITORY_ROOT, run)}/transfer.json`);
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
async function files(siteId: string): Promise<void> {
|
|
132
|
-
const { site, shared, sourceAppId } = await stacks(siteId);
|
|
133
|
-
const run = (await latest(siteId, "transfer.json")) ?? (await newRun(siteId));
|
|
134
|
-
const mapping = identityMapping(sourceAppId);
|
|
135
|
-
validateMapping(mapping);
|
|
136
|
-
const name = `${site.id}-tenant-move`;
|
|
137
|
-
const own = infrastructureFor(site, shared);
|
|
138
|
-
const config = buildCopyWorkerConfig(name, shared, own, WRANGLER_BASE.compatibility_date);
|
|
139
|
-
await writeFile(join(REPOSITORY_ROOT, COPY_WORKER_CONFIG), `${JSON.stringify({ ...config, name }, null, 2)}\n`, "utf8");
|
|
140
|
-
const token = randomBytes(32).toString("base64url");
|
|
141
|
-
console.log(`Deploying the short-lived copy Worker ${name}...`);
|
|
142
|
-
await runWrangler(COPY_WORKER_CONFIG, null, ["deploy"]);
|
|
143
|
-
try {
|
|
144
|
-
await runWrangler(COPY_WORKER_CONFIG, null, ["secret", "bulk"], { input: `${JSON.stringify({ COPY_TOKEN: token })}\n` });
|
|
145
|
-
const report = await copyFilesThrough(workerClient(`https://${name}.${own.workersDevDomain}`, token), mapping);
|
|
146
|
-
await writeFile(join(run, "files-report.json"), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 });
|
|
147
|
-
const total = report.jobs.reduce((sum, job) => sum + job.checked, 0);
|
|
148
|
-
console.log(`\nCopied and compared ${total} object(s), ${report.publications} publication(s). Report: ${relative(REPOSITORY_ROOT, run)}/files-report.json`);
|
|
149
|
-
} finally {
|
|
150
|
-
console.log(`Deleting the copy Worker ${name}...`);
|
|
151
|
-
await runWrangler(COPY_WORKER_CONFIG, null, ["delete", "--force"]);
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
export interface VerifyInput {
|
|
156
|
-
source: LogChecksum[];
|
|
157
|
-
target: LogChecksum[];
|
|
158
|
-
files: FilesReport | null;
|
|
159
|
-
/** A rehearsal against a local Ledger has no buckets to copy; a real move never skips the file check. */
|
|
160
|
-
skipFiles?: boolean;
|
|
161
|
-
/** What the destination answered to a request with a credential it did not issue. */
|
|
162
|
-
foreignCredentialStatus: number;
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
/** Everything `verify` checks, as a list of what is wrong. */
|
|
166
|
-
export function verificationProblems(input: VerifyInput): string[] {
|
|
167
|
-
const problems = sameChecksums(input.source, input.target);
|
|
168
|
-
if (!input.files && !input.skipFiles) problems.push("no files report: run the files step");
|
|
169
|
-
if (input.foreignCredentialStatus !== 401 && input.foreignCredentialStatus !== 403) {
|
|
170
|
-
problems.push(`the destination answered ${input.foreignCredentialStatus} to a credential it did not issue; it must refuse it`);
|
|
171
|
-
}
|
|
172
|
-
return problems;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
/** `--skip-files` is only for a rehearsal: the destination must be a Ledger on this machine. */
|
|
176
|
-
export function maySkipFiles(destinationUrl: string): boolean {
|
|
177
|
-
const host = new URL(destinationUrl).hostname;
|
|
178
|
-
return host === "localhost" || host === "127.0.0.1" || host === "[::1]" || host === "::1";
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
async function verificationProblemsFor(siteId: string, skipFiles: boolean): Promise<string[]> {
|
|
182
|
-
const { source, target, sourceAppId } = await stacks(siteId);
|
|
183
|
-
const destination = await target();
|
|
184
|
-
const read = async (call: AdminCall) => {
|
|
185
|
-
const response = await call("GET", `/admin/v1/apps/${sourceAppId}/checksums`);
|
|
186
|
-
if (!response.ok) throw new Error(`Reading checksums failed with ${response.status}`);
|
|
187
|
-
return ((await response.json()) as { logs: LogChecksum[] }).logs;
|
|
188
|
-
};
|
|
189
|
-
const run = await latest(siteId, "files-report.json");
|
|
190
|
-
const files = run ? (JSON.parse(await readFile(join(run, "files-report.json"), "utf8")) as FilesReport) : null;
|
|
191
|
-
return verificationProblems({ source: await read(source), target: await read(destination), files, skipFiles, foreignCredentialStatus: await foreignStatus(siteId) });
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
async function verify(siteId: string, skipFiles: boolean): Promise<void> {
|
|
195
|
-
if (skipFiles && !maySkipFiles(await destinationUrl(siteId))) {
|
|
196
|
-
throw new Error("--skip-files is for a rehearsal against a Ledger on localhost; the destination here is not local, so the file copy must be compared");
|
|
197
|
-
}
|
|
198
|
-
const problems = await verificationProblemsFor(siteId, skipFiles);
|
|
199
|
-
if (problems.length) {
|
|
200
|
-
console.error(`Verification failed:\n - ${problems.join("\n - ")}`);
|
|
201
|
-
process.exit(1);
|
|
202
|
-
}
|
|
203
|
-
console.log(skipFiles
|
|
204
|
-
? "Verified (files skipped, rehearsal): event checksums match on every log, and the destination refuses a credential it did not issue."
|
|
205
|
-
: "Verified: event checksums match on every log, the file copy was compared, and the destination refuses a credential it did not issue.");
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
/** The destination signs its own tokens: a credential it did not issue must be refused. */
|
|
209
|
-
async function destinationUrl(siteId: string): Promise<string> {
|
|
210
|
-
const { infrastructure, sites } = await loadTenantConfig();
|
|
211
|
-
const site = siteById(sites, siteId);
|
|
212
|
-
return (process.env[`LEDGER_BASE_URL_${site.id.toUpperCase().replace(/-/g, "_")}`]?.trim() || dedicatedLedgerBaseUrl(site, infrastructure)).replace(/\/$/, "");
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
async function foreignStatus(siteId: string): Promise<number> {
|
|
216
|
-
const base = await destinationUrl(siteId);
|
|
217
|
-
const response = await fetch(`${base}/session`, { headers: { Authorization: `Bearer ${randomBytes(48).toString("base64url")}` } });
|
|
218
|
-
return response.status;
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
const BIN = fileURLToPath(new URL("../../bin/hearthlog.mjs", import.meta.url));
|
|
222
|
-
|
|
223
|
-
/** Run a `hearthlog` command of this package for the site, and wait for it. */
|
|
224
|
-
function hearthlog(args: string[]): Promise<void> {
|
|
225
|
-
return new Promise((resolveRun, reject) => {
|
|
226
|
-
const child = spawn(process.execPath, [BIN, ...args], { cwd: REPOSITORY_ROOT, stdio: "inherit" });
|
|
227
|
-
child.on("error", reject);
|
|
228
|
-
child.on("exit", (code) => (code === 0 ? resolveRun() : reject(new Error(`hearthlog ${args.join(" ")} exited with ${String(code)}`))));
|
|
229
|
-
});
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
/**
|
|
233
|
-
* Write `tenants/<id>/deployment.json` for a site that is about to leave the shared stack, pinned to the shared stack's
|
|
234
|
-
* account so the move stays in it. This changes the site's configuration: it no longer deploys from the shared
|
|
235
|
-
* pipeline, and the shared feed Worker loses its binding. Keep the file out of anything that is pushed until the
|
|
236
|
-
* cutover is done (docs/tenant-split-runbook.md).
|
|
237
|
-
*/
|
|
238
|
-
async function declare(siteId: string, accountOverride: string | undefined): Promise<void> {
|
|
239
|
-
const { infrastructure, sites, stacks } = await loadTenantConfig();
|
|
240
|
-
const site = siteById(sites, siteId);
|
|
241
|
-
if (site.deployment) throw new Error(`${siteId} already has tenants/${siteId}/deployment.json`);
|
|
242
|
-
const accountId = accountOverride ?? infrastructure.accountId;
|
|
243
|
-
if (!accountId || !/^[0-9a-f]{32}$/.test(accountId)) {
|
|
244
|
-
throw new Error("Name the account: set infrastructure.accountId in tenants.json (the account of the shared stack) or pass --account-id");
|
|
245
|
-
}
|
|
246
|
-
const path = join(REPOSITORY_ROOT, "tenants", siteId, "deployment.json");
|
|
247
|
-
await writeFile(path, `${JSON.stringify({ schemaVersion: 1, resourcePrefix: `${siteId}-prod`, ledger: "dedicated", accountId }, null, 2)}\n`, { flag: "wx" });
|
|
248
|
-
console.log(`Wrote ${relative(REPOSITORY_ROOT, path)} (resource prefix ${siteId}-prod, account pinned).`);
|
|
249
|
-
console.log("Keep it local: pushing it would redeploy the shared feed Worker without this site's binding while its frontend still runs on the shared stack.");
|
|
250
|
-
if (gallerySweeperId(stacks, infrastructure) === siteId) {
|
|
251
|
-
console.log(`${siteId} sweeps the shared public Gallery bucket: set infrastructure.gallerySweeper in tenants.json to a site that stays on the shared stack.`);
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
/**
|
|
256
|
-
* The switch: check the move again, mint the gateway's token on the new Ledger, and deploy the gateway bound to it. The
|
|
257
|
-
* token goes to the live gateway first, which still talks to the old Ledger until the deploy lands, so the two run
|
|
258
|
-
* back to back. It stops before marking the old application moved (`fence --mode moved`).
|
|
259
|
-
*/
|
|
260
|
-
async function cutover(siteId: string, skipAccess: boolean): Promise<void> {
|
|
261
|
-
const problems = await verificationProblemsFor(siteId, false);
|
|
262
|
-
if (problems.length) {
|
|
263
|
-
console.error(`Not cutting over; verification failed:\n - ${problems.join("\n - ")}`);
|
|
264
|
-
process.exit(1);
|
|
265
|
-
}
|
|
266
|
-
console.log("Verified. Minting the gateway token on the new Ledger, then deploying the gateway.");
|
|
267
|
-
// The Access application is the same one (same origin), so a run without CLOUDFLARE_API_TOKEN may leave it alone;
|
|
268
|
-
// the gateway keeps the audience secret it already holds.
|
|
269
|
-
await hearthlog(["ensure", "--site", siteId, "--rotate-token", ...(skipAccess ? ["--skip-access"] : [])]);
|
|
270
|
-
await hearthlog(["deploy", "--site", siteId]);
|
|
271
|
-
console.log(`\nThe ${siteId} gateway now talks to its own Ledger. Next: pnpm tenant:split -- fence --site ${siteId} --mode moved`);
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
async function main(argv: string[]): Promise<void> {
|
|
275
|
-
const args = argv.filter((argument) => argument !== "--");
|
|
276
|
-
const [command, ...rest] = args;
|
|
277
|
-
const value = (name: string) => {
|
|
278
|
-
const index = rest.indexOf(name);
|
|
279
|
-
return index === -1 ? undefined : rest[index + 1]?.trim();
|
|
280
|
-
};
|
|
281
|
-
const siteId = value("--site");
|
|
282
|
-
if (!command || !siteId) {
|
|
283
|
-
console.error("Usage: pnpm tenant:split -- <declare|fence|transfer|files|verify|cutover> --site <id> [--mode tenant_migrating|moved|off] [--allow-unfenced] [--skip-files] [--skip-access] [--account-id <id>]");
|
|
284
|
-
process.exit(1);
|
|
285
|
-
}
|
|
286
|
-
if (command === "declare") await declare(siteId, value("--account-id"));
|
|
287
|
-
else if (command === "fence") await fence(siteId, value("--mode") ?? "");
|
|
288
|
-
else if (command === "transfer") await transfer(siteId, rest.includes("--allow-unfenced"));
|
|
289
|
-
else if (command === "files") await files(siteId);
|
|
290
|
-
else if (command === "verify") await verify(siteId, rest.includes("--skip-files"));
|
|
291
|
-
else if (command === "cutover") await cutover(siteId, rest.includes("--skip-access"));
|
|
292
|
-
else throw new Error(`Unknown command "${command}"`);
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main(process.argv.slice(2));
|
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* What a tenant copy moves between buckets, as pure data.
|
|
3
|
-
*
|
|
4
|
-
* A copy gives the new tenant new app and log IDs, and private object keys
|
|
5
|
-
* embed them: invoice files are `<prefix>/<appId>/<logId>/…` and Gallery
|
|
6
|
-
* files `v1/<appId>/…`. Gallery publications are keyed by their own ID, but
|
|
7
|
-
* their `owner.json` names the app and log. The generated-invoice cache
|
|
8
|
-
* (`v1/<appId>/<logId>/invoices/…`) is not copied: it rebuilds on demand.
|
|
9
|
-
* Calendar feeds are not copied: the new tenant creates its own.
|
|
10
|
-
*
|
|
11
|
-
* A **move** (the split of a shared tenant onto its own stack) keeps the app
|
|
12
|
-
* and log IDs, so every key stays exactly as it is: `identity: true` copies
|
|
13
|
-
* each prefix to the same name in the destination bucket, per app rather than
|
|
14
|
-
* per log, and publication owner records are already correct and left alone.
|
|
15
|
-
*/
|
|
16
|
-
|
|
17
|
-
export interface TenantCopyMapping {
|
|
18
|
-
sourceAppId: string;
|
|
19
|
-
targetAppId: string;
|
|
20
|
-
/** Source log ID to target log ID, for every copied log. Empty for an identity move. */
|
|
21
|
-
logs: Readonly<Record<string, string>>;
|
|
22
|
-
/** The destination keeps the app and log IDs, so keys are copied unchanged. */
|
|
23
|
-
identity?: true;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/** The mapping of a move that keeps every identity. */
|
|
27
|
-
export function identityMapping(appId: string): TenantCopyMapping {
|
|
28
|
-
return { sourceAppId: appId, targetAppId: appId, logs: {}, identity: true };
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
export type BucketPair = "invoice-pdfs" | "gallery-files" | "gallery-public";
|
|
32
|
-
|
|
33
|
-
export interface CopyJob {
|
|
34
|
-
pair: BucketPair;
|
|
35
|
-
fromPrefix: string;
|
|
36
|
-
toPrefix: string;
|
|
37
|
-
/** Rewrite `owner.json` objects under this prefix to the target app and log. */
|
|
38
|
-
rewriteOwner?: boolean;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
const INVOICE_FILE_PREFIXES = ["received/v1", "received-isdoc/v1", "issued/v1", "issued-isdoc/v1"] as const;
|
|
42
|
-
|
|
43
|
-
const ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
44
|
-
|
|
45
|
-
function requireId(value: string, label: string): void {
|
|
46
|
-
if (!ID.test(value)) throw new Error(`${label} is not a UUID: ${value}`);
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
export function validateMapping(mapping: TenantCopyMapping): void {
|
|
50
|
-
requireId(mapping.sourceAppId, "source app");
|
|
51
|
-
requireId(mapping.targetAppId, "target app");
|
|
52
|
-
if (mapping.identity) {
|
|
53
|
-
if (mapping.sourceAppId !== mapping.targetAppId) throw new Error("an identity move keeps the app: source and target must be the same id");
|
|
54
|
-
if (Object.keys(mapping.logs).length) throw new Error("an identity move names no logs: every log keeps its id");
|
|
55
|
-
return;
|
|
56
|
-
}
|
|
57
|
-
if (mapping.sourceAppId === mapping.targetAppId) throw new Error("source and target apps are the same");
|
|
58
|
-
for (const [source, target] of Object.entries(mapping.logs)) {
|
|
59
|
-
requireId(source, "source log");
|
|
60
|
-
requireId(target, "target log");
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
/** Every prefix to copy, given the publications the source app owns. */
|
|
65
|
-
export function planFileCopies(mapping: TenantCopyMapping, ownedPublications: readonly string[]): CopyJob[] {
|
|
66
|
-
validateMapping(mapping);
|
|
67
|
-
const jobs: CopyJob[] = [];
|
|
68
|
-
if (mapping.identity) {
|
|
69
|
-
for (const prefix of INVOICE_FILE_PREFIXES) {
|
|
70
|
-
const key = `${prefix}/${mapping.sourceAppId}/`;
|
|
71
|
-
jobs.push({ pair: "invoice-pdfs", fromPrefix: key, toPrefix: key });
|
|
72
|
-
}
|
|
73
|
-
jobs.push({ pair: "gallery-files", fromPrefix: `v1/${mapping.sourceAppId}/`, toPrefix: `v1/${mapping.sourceAppId}/` });
|
|
74
|
-
for (const publication of ownedPublications) {
|
|
75
|
-
requireId(publication, "publication");
|
|
76
|
-
const control = `v1/publications/${publication}/`;
|
|
77
|
-
jobs.push({ pair: "gallery-files", fromPrefix: control, toPrefix: control });
|
|
78
|
-
jobs.push({ pair: "gallery-public", fromPrefix: `v1/galleries/${publication}/`, toPrefix: `v1/galleries/${publication}/` });
|
|
79
|
-
}
|
|
80
|
-
return jobs;
|
|
81
|
-
}
|
|
82
|
-
for (const prefix of INVOICE_FILE_PREFIXES) {
|
|
83
|
-
for (const [sourceLog, targetLog] of Object.entries(mapping.logs)) {
|
|
84
|
-
jobs.push({
|
|
85
|
-
pair: "invoice-pdfs",
|
|
86
|
-
fromPrefix: `${prefix}/${mapping.sourceAppId}/${sourceLog}/`,
|
|
87
|
-
toPrefix: `${prefix}/${mapping.targetAppId}/${targetLog}/`,
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
jobs.push({ pair: "gallery-files", fromPrefix: `v1/${mapping.sourceAppId}/`, toPrefix: `v1/${mapping.targetAppId}/` });
|
|
92
|
-
for (const publication of ownedPublications) {
|
|
93
|
-
requireId(publication, "publication");
|
|
94
|
-
jobs.push({
|
|
95
|
-
pair: "gallery-files",
|
|
96
|
-
fromPrefix: `v1/publications/${publication}/`,
|
|
97
|
-
toPrefix: `v1/publications/${publication}/`,
|
|
98
|
-
rewriteOwner: true,
|
|
99
|
-
});
|
|
100
|
-
jobs.push({ pair: "gallery-public", fromPrefix: `v1/galleries/${publication}/`, toPrefix: `v1/galleries/${publication}/` });
|
|
101
|
-
}
|
|
102
|
-
return jobs;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
export function isOwnerRecord(key: string): boolean {
|
|
106
|
-
return /^v1\/publications\/[^/]+\/owner\.json$/.test(key);
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/** The owner record for the target tenant; throws for a log the copy left out. */
|
|
110
|
-
export function rewriteOwner(record: unknown, mapping: TenantCopyMapping): { appId: string; logId: string } {
|
|
111
|
-
const owner = record as { appId?: unknown; logId?: unknown } | null;
|
|
112
|
-
if (!owner || typeof owner.appId !== "string" || typeof owner.logId !== "string") {
|
|
113
|
-
throw new Error("publication owner record is malformed");
|
|
114
|
-
}
|
|
115
|
-
if (owner.appId !== mapping.sourceAppId) throw new Error("publication belongs to another app");
|
|
116
|
-
const logId = mapping.logs[owner.logId];
|
|
117
|
-
if (!logId) throw new Error(`publication belongs to log ${owner.logId}, which was not copied`);
|
|
118
|
-
return { appId: mapping.targetAppId, logId };
|
|
119
|
-
}
|