@runeditrun/rer 0.2.58 → 0.2.60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/cli/export.mjs +8 -3
  2. package/default-seed/public/brand/mark.svg +1 -0
  3. package/default-seed/public/fonts/outfit/LICENSE.txt +93 -0
  4. package/default-seed/public/fonts/outfit/outfit-latin-ext-wght-normal.woff2 +0 -0
  5. package/default-seed/public/fonts/outfit/outfit-latin-wght-normal.woff2 +0 -0
  6. package/default-seed/public/rer-theme.css +24 -0
  7. package/global/CHANGELOG.md +10 -0
  8. package/modules/demo-host/README.md +6 -0
  9. package/modules/demo-host/native/loader.mjs +9 -4
  10. package/modules/demo-host/native/prepare.mjs +5 -2
  11. package/modules/demo-host/native/visitor.mjs +3 -3
  12. package/modules/demo-host/native/worker.mjs +1 -1
  13. package/modules/demo-host/page-style.mjs +19 -0
  14. package/modules/demo-host/pages.mjs +5 -22
  15. package/modules/demo-host/plan.mjs +1 -1
  16. package/modules/demo-host/theme-script.mjs +18 -0
  17. package/modules/demo-host/theme.mjs +27 -0
  18. package/modules/portability/README.md +40 -12
  19. package/modules/portability/bookkeeping.mjs +25 -0
  20. package/modules/portability/catalogue.mjs +12 -13
  21. package/modules/portability/cli.mjs +2 -0
  22. package/modules/portability/files.mjs +6 -2
  23. package/modules/portability/import.mjs +8 -4
  24. package/modules/portability/index.d.mts +2 -2
  25. package/modules/portability/progress.mjs +2 -0
  26. package/modules/release/produce.mjs +4 -23
  27. package/modules/release/publication-scan.mjs +36 -4
  28. package/package.json +1 -1
  29. package/scripts/publish-check/README.md +14 -0
  30. package/scripts/publish-check/cli.py +2 -2
  31. package/scripts/publish-check/git_history.py +3 -0
  32. package/scripts/publish-check/gitleaks.py +1 -1
  33. package/scripts/publish-check/history_hydration.py +28 -0
  34. package/scripts/publish-check/history_scan.py +25 -16
  35. package/scripts/publish-check/scan_timings.py +46 -0
  36. package/scripts/publish-check/staged_annotation.py +39 -0
  37. package/scripts/publish-check.mjs +3 -0
package/cli/export.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { cp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises';
1
+ import { cp, mkdir, readFile, readdir, rm, writeFile, lstat } from 'node:fs/promises';
2
2
  import { createHash } from 'node:crypto';
3
3
  import { resolve, dirname, relative } from 'node:path';
4
4
  import { testSource } from '../composition/git-source.mjs';
@@ -6,7 +6,7 @@ import { cliVersion } from './version.mjs';
6
6
 
7
7
  // Copy whole maintained source/tooling directories. Python publication scans
8
8
  // load sibling modules and data dynamically; they belong in the inventory too.
9
- export const sourceRoots = ['bin/rer.mjs', 'cli', 'modules/delivery-producer/declarations.mjs', 'modules/delivery-producer/bindings.mjs', 'modules/delivery-producer/migrations.mjs', 'modules/delivery-producer/mail.mjs', 'modules/job-queue', 'modules/release', 'modules/deploy-kit', 'modules/container', 'modules/demo-host', 'modules/demo', 'modules/integrations/core', 'modules/mail', 'modules/ai', 'modules/portability', 'modules/d1-transport', 'modules/erasure', 'modules/dependencies', 'modules/operator-auth', 'modules/owner-claim', 'operations', 'scripts/publish-check', 'scripts/publish-check.mjs', 'composition/git-source.mjs', 'edition-authoring/manifest.mjs', 'edition-authoring/files.mjs', 'foundation/runtime/seed-contract.mjs'];
9
+ export const sourceRoots = ['default-seed/public/rer-theme.css', 'default-seed/public/brand/mark.svg', 'default-seed/public/fonts/outfit', 'bin/rer.mjs', 'cli', 'modules/delivery-producer/declarations.mjs', 'modules/delivery-producer/bindings.mjs', 'modules/delivery-producer/migrations.mjs', 'modules/delivery-producer/mail.mjs', 'modules/job-queue', 'modules/release', 'modules/deploy-kit', 'modules/container', 'modules/demo-host', 'modules/demo', 'modules/integrations/core', 'modules/mail', 'modules/ai', 'modules/portability', 'modules/d1-transport', 'modules/erasure', 'modules/dependencies', 'modules/operator-auth', 'modules/owner-claim', 'operations', 'scripts/publish-check', 'scripts/publish-check.mjs', 'composition/git-source.mjs', 'edition-authoring/manifest.mjs', 'edition-authoring/files.mjs', 'foundation/runtime/seed-contract.mjs'];
10
10
  export async function inventory(base, roots = sourceRoots) {
11
11
  const files = [];
12
12
  async function walk(path) {
@@ -19,7 +19,12 @@ export async function inventory(base, roots = sourceRoots) {
19
19
  else throw Error('Non-file CLI export source');
20
20
  }
21
21
  }
22
- for (const source of roots) { if (source.endsWith('.mjs')) files.push(source); else await walk(source); }
22
+ for (const source of roots) {
23
+ const stat = await lstat(resolve(base, source));
24
+ if (stat.isFile()) files.push(source);
25
+ else if (stat.isDirectory()) await walk(source);
26
+ else throw Error('Non-file CLI export source');
27
+ }
23
28
  return files.sort();
24
29
  }
25
30
 
@@ -0,0 +1 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32"><rect width="32" height="32" rx="8" fill="#334155"/><path d="M16 25V14m0 0c0-4 3-6 7-6 0 4-3 6-7 6Zm0 3c0-3-2.5-5-6-5 0 3 2.5 5 6 5Z" fill="none" stroke="#fff" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/></svg>
@@ -0,0 +1,93 @@
1
+ Copyright 2021 The Outfit Project Authors (https://github.com/Outfitio/Outfit-Fonts)
2
+
3
+ This Font Software is licensed under the SIL Open Font License, Version 1.1.
4
+ This license is copied below, and is also available with a FAQ at:
5
+ http://scripts.sil.org/OFL
6
+
7
+
8
+ -----------------------------------------------------------
9
+ SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
10
+ -----------------------------------------------------------
11
+
12
+ PREAMBLE
13
+ The goals of the Open Font License (OFL) are to stimulate worldwide
14
+ development of collaborative font projects, to support the font creation
15
+ efforts of academic and linguistic communities, and to provide a free and
16
+ open framework in which fonts may be shared and improved in partnership
17
+ with others.
18
+
19
+ The OFL allows the licensed fonts to be used, studied, modified and
20
+ redistributed freely as long as they are not sold by themselves. The
21
+ fonts, including any derivative works, can be bundled, embedded,
22
+ redistributed and/or sold with any software provided that any reserved
23
+ names are not used by derivative works. The fonts and derivatives,
24
+ however, cannot be released under any other type of license. The
25
+ requirement for fonts to remain under this license does not apply
26
+ to any document created using the fonts or their derivatives.
27
+
28
+ DEFINITIONS
29
+ "Font Software" refers to the set of files released by the Copyright
30
+ Holder(s) under this license and clearly marked as such. This may
31
+ include source files, build scripts and documentation.
32
+
33
+ "Reserved Font Name" refers to any names specified as such after the
34
+ copyright statement(s).
35
+
36
+ "Original Version" refers to the collection of Font Software components as
37
+ distributed by the Copyright Holder(s).
38
+
39
+ "Modified Version" refers to any derivative made by adding to, deleting,
40
+ or substituting -- in part or in whole -- any of the components of the
41
+ Original Version, by changing formats or by porting the Font Software to a
42
+ new environment.
43
+
44
+ "Author" refers to any designer, engineer, programmer, technical
45
+ writer or other person who contributed to the Font Software.
46
+
47
+ PERMISSION & CONDITIONS
48
+ Permission is hereby granted, free of charge, to any person obtaining
49
+ a copy of the Font Software, to use, study, copy, merge, embed, modify,
50
+ redistribute, and sell modified and unmodified copies of the Font
51
+ Software, subject to the following conditions:
52
+
53
+ 1) Neither the Font Software nor any of its individual components,
54
+ in Original or Modified Versions, may be sold by itself.
55
+
56
+ 2) Original or Modified Versions of the Font Software may be bundled,
57
+ redistributed and/or sold with any software, provided that each copy
58
+ contains the above copyright notice and this license. These can be
59
+ included either as stand-alone text files, human-readable headers or
60
+ in the appropriate machine-readable metadata fields within text or
61
+ binary files as long as those fields can be easily viewed by the user.
62
+
63
+ 3) No Modified Version of the Font Software may use the Reserved Font
64
+ Name(s) unless explicit written permission is granted by the corresponding
65
+ Copyright Holder. This restriction only applies to the primary font name as
66
+ presented to the users.
67
+
68
+ 4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
69
+ Software shall not be used to promote, endorse or advertise any
70
+ Modified Version, except to acknowledge the contribution(s) of the
71
+ Copyright Holder(s) and the Author(s) or with their explicit written
72
+ permission.
73
+
74
+ 5) The Font Software, modified or unmodified, in part or in whole,
75
+ must be distributed entirely under this license, and must not be
76
+ distributed under any other license. The requirement for fonts to
77
+ remain under this license does not apply to any document created
78
+ using the Font Software.
79
+
80
+ TERMINATION
81
+ This license becomes null and void if any of the above conditions are
82
+ not met.
83
+
84
+ DISCLAIMER
85
+ THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
86
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
87
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
88
+ OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
89
+ COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
90
+ INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
91
+ DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
92
+ FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
93
+ OTHER DEALINGS IN THE FONT SOFTWARE.
@@ -0,0 +1,24 @@
1
+ /* Default seed's declared theme. The installer, sign-in confirmation, /setup
2
+ read these --rer-* tokens; the app serves this file and its fonts. */
3
+ @font-face {
4
+ font-family: Outfit;
5
+ font-style: normal;
6
+ font-weight: 100 900;
7
+ font-display: swap;
8
+ src: url('/fonts/outfit/outfit-latin-wght-normal.woff2') format('woff2');
9
+ }
10
+ :root {
11
+ --rer-font-body: Outfit, Arial, sans-serif;
12
+ --rer-font-heading: Outfit, Arial, sans-serif;
13
+ --rer-page: #f3f5f8;
14
+ --rer-surface: #ffffff;
15
+ --rer-text: #172033;
16
+ --rer-muted: #475569;
17
+ --rer-border: #64748b;
18
+ --rer-focus: #172033;
19
+ --rer-accent: #334155;
20
+ --rer-accent-text: #ffffff;
21
+ --rer-link: #1d4ed8;
22
+ --rer-radius: 16px;
23
+ --rer-logo: url('/brand/mark.svg');
24
+ }
@@ -1,5 +1,15 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.60
4
+
5
+ - **Hosted demo gates wear each seed’s theme.** Start and ended pages load the seed’s mark, palette, fonts and focus styles, with Base theme assets when absent (#284).
6
+ - **Portability leaves platform bookkeeping behind.** Exports never carry rer-owned tables such as `rer_migrations`, and `includeTables` cannot re-add them. Fresh migrated targets can restore seed data; old archives keep their integrity checks while skipping platform rows with one clear CLI line (#285).
7
+
8
+ ## 0.2.59
9
+
10
+ - **Release publication scans complete on large seeds and partial clones.** `rer release` hydrates missing history blobs in one fetch and inspects the staged release annotation in the same scan. Orbit finishes in 82 seconds instead of exceeding the 300-second budget, with identical findings. A seed must adopt the new rer through its pin for its release command to use it (#278).
11
+ - **Native demos decode binary modules when needed.** WebAssembly and data modules decode lazily inside the Worker Loader factory with native base64 support, reducing host startup work while preserving their bytes (#280).
12
+
3
13
  ## 0.2.58
4
14
 
5
15
  - **Deploy dry runs validate required target secrets.** `rer deploy --dry-run` uses the real deploy secret-binding resolver and refuses with the same exit and message when required bindings are missing. It names every missing binding and its environment input, accepts secrets already held by the Worker, and makes no target writes. Missing credentials or read permission also refuse (#275).
@@ -12,6 +12,12 @@ host. See [adoption and deployment](../../export/DEMO.md#hosting-the-demo).
12
12
  outbound requests return to the same visitor, including declared secondary hosts.
13
13
  - `worker.mjs`, `pages.mjs`, `start-script.mjs`: Turnstile start, progress,
14
14
  cancellation and visitor routing. Static assets are served at the edge.
15
+ - `theme.mjs`, `page-style.mjs`, `theme-script.mjs`: the start and ended pages
16
+ link the seed's built `rer-theme.css` and fonts, as the Lobby does. Their
17
+ colours, type and mark follow its `--rer-*` tokens, including media rules.
18
+ The maintained default seed's theme assets supply missing tokens or an
19
+ absent theme, under `/__demo/theme/`. Styles, fonts and marks stay same-origin;
20
+ a missing mark leaves the product name visible.
15
21
  - `native/warm.mjs`: the start page's prefetch list, built from `dist/client`
16
22
  (the Vite manifest if the build wrote one, else the build's `assetsDir`, else
17
23
  `assets`) and TanStack Start's route manifest, the console's views first,
@@ -1,16 +1,21 @@
1
1
  import { WorkerEntrypoint } from 'cloudflare:workers';
2
+ import { Buffer } from 'node:buffer';
2
3
  import demoEgress from '../../container/demo-egress.mjs';
3
4
  import bundle from './bundle.mjs';
4
5
 
5
- // wasm and data modules arrive as base64; the Loader takes their bytes.
6
- const bytes = text => Uint8Array.from(atob(text), char => char.charCodeAt(0)).buffer;
7
- const modules = Object.fromEntries(Object.entries(bundle.modules).map(([name, module]) =>
6
+ // Decode only when a Loader factory needs the modules, once per host isolate.
7
+ const bytes = text => {
8
+ const buffer = Buffer.from(text, 'base64');
9
+ return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
10
+ };
11
+ let modules;
12
+ const loaderModules = () => modules ??= Object.fromEntries(Object.entries(bundle.modules).map(([name, module]) =>
8
13
  [name, 'wasm' in module ? { wasm: bytes(module.wasm) } : 'data' in module ? { data: bytes(module.data) } : module]));
9
14
 
10
15
  export function loaded(env, ctx, id, runtime) {
11
16
  return env.LOADER.get(`${bundle.digest}:${id}`, () => ({
12
17
  compatibilityDate: bundle.compatibilityDate, compatibilityFlags: bundle.compatibilityFlags,
13
- mainModule: '_rer/isolate.js', modules,
18
+ mainModule: '_rer/isolate.js', modules: loaderModules(),
14
19
  globalOutbound: ctx.exports.DemoEgress({ props: { id } }),
15
20
  env: { ...runtime, RER_NATIVE_CONFIG: bundle.bindings, RER_NATIVE_HOST: ctx.exports.HostBridge({ props: { id, key: runtime.RER_NATIVE_KEY } }), [bundle.bindings.assets]: ctx.exports.DemoAssets({}) },
16
21
  }));
@@ -5,6 +5,7 @@ import { collectBuildModules } from '../../deploy-kit/build-modules.mjs';
5
5
  import { collectFiles, moduleType } from '../../deploy-kit/artifact.mjs';
6
6
  import { nativeBindings, ISOLATE_FILES, objectExports } from './config.mjs';
7
7
  import { warmList, builtChunks } from './warm.mjs';
8
+ import { prepareTheme } from '../theme.mjs';
8
9
 
9
10
  // The Worker Loader module kind for each upload type deploy-kit gives a build
10
11
  // module, so a demo carries what a normal deploy uploads. Binary kinds travel
@@ -31,13 +32,15 @@ export async function prepareNative(folder) {
31
32
  + objectExports(bindings) };
32
33
  for (const file of ISOLATE_FILES) modules[`_rer/${file}.js`] = { js: await readFile(new URL(`./${file}.mjs`, import.meta.url), 'utf8') };
33
34
  const assets = await collectFiles(join(app, 'dist/client'));
35
+ await cp(join(app, 'dist/client'), join(folder, 'assets'), { recursive: true });
36
+ const theme = await prepareTheme(folder, assets);
34
37
  const bundle = { compatibilityDate: config.compatibility_date, compatibilityFlags: [...new Set([...(config.compatibility_flags ?? []), 'nodejs_compat'])], modules,
35
38
  bindings, migrations: manifest.migrations, vars: config.vars ?? {}, crons: config.triggers?.crons ?? [],
36
- assetPaths: assets.filter(file => !file.path.split('/').some(part => part.startsWith('.'))).map(file => '/' + file.path),
39
+ assetPaths: [...assets.filter(file => !file.path.split('/').some(part => part.startsWith('.'))).map(file => '/' + file.path), ...theme.paths],
40
+ theme: theme.stylesheets,
37
41
  immutable: [...builtChunks(assets, manifest.assetsDir)].sort(),
38
42
  warm: (await warmList(assets, root, manifest.operatorPath, manifest.assetsDir)).paths };
39
43
  bundle.digest = createHash('sha256').update(JSON.stringify(bundle)).digest('hex');
40
- await cp(join(app, 'dist/client'), join(folder, 'assets'), { recursive: true });
41
44
  await writeFile(join(folder, 'modules/demo-host/native/bundle.mjs'), `export default ${JSON.stringify(bundle)};\n`);
42
45
  return bundle;
43
46
  }
@@ -98,7 +98,7 @@ export class NativeDemo extends DurableObject {
98
98
  return this.object(binding, id, 'fetch', [new Request(request, { headers })]);
99
99
  }
100
100
  const runtime = await this.session();
101
- if (!runtime) return endedPage(this.env, { headers: { 'set-cookie': clearCookie(this.env) } });
101
+ if (!runtime) return endedPage({ ...this.env, DEMO_THEME: bundle.theme }, { headers: { 'set-cookie': clearCookie(this.env) } });
102
102
  return application(this.env, this.ctx, this.ctx.id.toString(), runtime).fetch(request);
103
103
  }
104
104
  // The seed signs the visitor out, then the copy ends as an expired one does.
@@ -106,12 +106,12 @@ export class NativeDemo extends DurableObject {
106
106
  // its next page load finds the ended page.
107
107
  async signOut(request) {
108
108
  const runtime = await this.session();
109
- if (!runtime) return endedPage(this.env, { headers: { 'set-cookie': clearCookie(this.env) } });
109
+ if (!runtime) return endedPage({ ...this.env, DEMO_THEME: bundle.theme }, { headers: { 'set-cookie': clearCookie(this.env) } });
110
110
  const response = await application(this.env, this.ctx, this.ctx.id.toString(), runtime).fetch(request);
111
111
  if (response.status >= 400) return response;
112
112
  const body = await response.arrayBuffer();
113
113
  await this.end();
114
- if (/text\/html/.test(request.headers.get('accept') ?? '')) return endedPage(this.env, { headers: { 'set-cookie': clearCookie(this.env) } });
114
+ if (/text\/html/.test(request.headers.get('accept') ?? '')) return endedPage({ ...this.env, DEMO_THEME: bundle.theme }, { headers: { 'set-cookie': clearCookie(this.env) } });
115
115
  return new Response(response.status === 204 ? null : body, response);
116
116
  }
117
117
  database(binding, method, args) { return database(this, binding, method, args); }
@@ -21,6 +21,6 @@ export default {
21
21
  return new Response(response.body, { status: response.status, headers });
22
22
  }
23
23
  // The start page prefetches the console's chunks while the visitor waits.
24
- return front.fetch(request, { ...env, DEMO_WARM: bundle.warm }, ctx);
24
+ return front.fetch(request, { ...env, DEMO_WARM: bundle.warm, DEMO_THEME: bundle.theme }, ctx);
25
25
  },
26
26
  };
@@ -0,0 +1,19 @@
1
+ // Gate layout; presentation comes from the same --rer-* tokens as the Lobby.
2
+ export const STYLE = `*{box-sizing:border-box}body{margin:0;background:var(--rer-page);color:var(--rer-text);font-family:var(--rer-font-body);-webkit-font-smoothing:antialiased}
3
+ main{width:666px;margin:clamp(32px,18vh,184px) auto 48px}.card{padding:40px 42px 44px;background:var(--rer-surface);border:1px solid var(--rer-border);border-radius:var(--rer-radius);box-shadow:0 12px 45px color-mix(in srgb,var(--rer-text) 5%,transparent)}
4
+ .brand{display:flex;align-items:center;gap:12px;font-family:var(--rer-font-heading,var(--rer-font-body));font-size:18px;line-height:22px;font-weight:var(--rer-wordmark-weight,600);letter-spacing:var(--rer-wordmark-tracking,normal);color:var(--rer-text);margin:0 0 20px}.brand img{display:block;width:40px;height:40px;object-fit:contain}.brand img[hidden]{display:none}
5
+ h1{font-family:var(--rer-font-heading,var(--rer-font-body));font-size:42px;line-height:50px;letter-spacing:-1.3px;margin:0 0 14px;font-weight:700}.intro{font-size:20px;line-height:28px;color:var(--rer-muted);margin:0}
6
+ ul{padding:0;margin:23px 0 35px;list-style:none;color:var(--rer-muted);font-size:18px;line-height:33px}li{position:relative;padding-left:33px}li:before{content:'';position:absolute;left:7px;top:14px;width:7px;height:7px;background:var(--rer-accent);border-radius:50%}
7
+ .notice{margin:0 0 16px;padding:12px 16px;border-radius:8px;background:var(--rer-page);color:var(--rer-text);border:1px solid var(--rer-border);font-size:16px;line-height:22px}
8
+ .cf-turnstile{min-height:65px}.button{display:flex;align-items:center;justify-content:center;gap:12px;width:100%;margin-top:28px;min-height:58px;border:1px solid var(--rer-accent);border-radius:var(--rer-button-radius,var(--rer-radius));background:var(--rer-accent);color:var(--rer-accent-text);font:inherit;font-size:20px;font-weight:700;line-height:26px;padding:15px;text-align:center;text-decoration:none;cursor:pointer}
9
+ .button:disabled{cursor:progress}[data-state=waiting] .button{font-weight:600;cursor:default}
10
+ .spinner{display:none;flex:none;width:20px;height:20px;border:3px solid color-mix(in srgb,currentColor 35%,transparent);border-top-color:currentColor;border-radius:50%;animation:spin .8s linear infinite}[data-state=working] .spinner,[data-state=signing] .spinner{display:block}@keyframes spin{to{transform:rotate(360deg)}}
11
+ ol.steps{display:none;list-style:none;margin:22px auto 0;padding:0;width:max-content;max-width:100%;font-size:16px;line-height:22px;color:var(--rer-muted)}[data-state=working] .steps,[data-state=signing] .steps,[data-state=slow] .steps{display:grid;gap:10px}[data-state=failed] .note{display:none}
12
+ .steps li{padding-left:30px;margin:0}.steps li:before{left:0;top:2px;width:18px;height:18px;box-sizing:border-box;background:none;border:2px solid var(--rer-muted)}.steps [data-at=done]{color:var(--rer-muted)}.steps [data-at=done]:before{content:'✓';border:0;background:var(--rer-accent);color:var(--rer-accent-text);font-size:12px;line-height:18px;font-weight:700;text-align:center}.steps [data-at=now]{color:var(--rer-text);font-weight:600}.steps [data-at=now]:before{border-color:var(--rer-border);border-top-color:var(--rer-accent);animation:spin .8s linear infinite}
13
+ .visually-hidden{position:absolute;width:1px;height:1px;margin:-1px;overflow:hidden;clip:rect(0 0 0 0);white-space:nowrap}
14
+ @media(prefers-reduced-motion:reduce){.spinner,.steps [data-at=now]:before{animation:none}.steps [data-at=now]:before{border-color:var(--rer-accent)}}.button:focus-visible,a:focus-visible{outline:3px solid var(--rer-focus);outline-offset:4px}
15
+ .note{font-size:15px;line-height:21px;color:var(--rer-muted);text-align:center;margin:21px 0 0}.secondary{text-align:center;margin:22px 0 0;font-size:18px;line-height:24px;font-weight:600}.secondary a{color:var(--rer-link,var(--rer-text));text-underline-offset:3px}
16
+ .back{text-align:center;margin:32px 0 0;font-size:15px;line-height:22px}.back a{color:var(--rer-link,var(--rer-text));text-underline-offset:3px}
17
+ .ended main{width:min(560px,calc(100% - 32px));margin-top:clamp(32px,26vh,262px)}.ended .card{padding:28px 36px 42px}.ended .brand{font-size:16px;margin-bottom:26px}.ended h1{font-size:36px;line-height:44px;margin-bottom:22px}.ended .intro{margin-bottom:36px}.ended .button{margin-top:0;font-size:18px}
18
+ @media(min-width:600px) and (max-width:1023px){main{width:calc(100% - 64px);max-width:666px;margin:100px auto 48px}.card{padding:36px}h1{font-size:38px;line-height:46px}.intro{font-size:18px;line-height:27px}}
19
+ @media(max-width:599px){main,.ended main{width:calc(100% - 32px);margin:32px auto}.ended .card{padding:26px 22px}.ended h1{font-size:29px;line-height:36px}.ended .intro{font-size:17px;line-height:25px;margin-bottom:28px}.card{padding:26px 22px;border-radius:var(--rer-radius)}.brand{font-size:16px}h1{font-size:34px;line-height:41px;letter-spacing:-.8px}.intro{font-size:17px;line-height:25px}ul{font-size:16px;line-height:25px;margin:22px 0 26px}li{padding-left:22px;margin-bottom:9px}li:before{left:2px;top:9px}.button{font-size:18px;margin-top:22px}.note{font-size:13px;line-height:20px}.steps{font-size:15px}.back{margin-top:24px;font-size:14px}}`;
@@ -2,28 +2,11 @@
2
2
  // Turnstile managed widget starts the demo by itself, start-script.mjs) and the ended page.
3
3
  // Both are one card in the family's quiet sign-in style.
4
4
  import { START_SCRIPT } from './start-script.mjs';
5
+ import { STYLE } from './page-style.mjs';
6
+ import { LOGO_SCRIPT } from './theme-script.mjs';
5
7
 
6
8
  const escape = value => String(value).replace(/[&<>"']/g, character => `&#${character.charCodeAt(0)};`);
7
9
 
8
- const STYLE = `*{box-sizing:border-box}body{margin:0;background:#f2f5f7;color:#192333;font-family:system-ui,-apple-system,'Segoe UI',Arial,sans-serif;-webkit-font-smoothing:antialiased}
9
- main{width:666px;margin:clamp(32px,18vh,184px) auto 48px}.card{padding:40px 42px 44px;background:#fff;border:1px solid #d6dfe9;border-radius:18px;box-shadow:0 12px 45px rgba(42,65,86,.05)}
10
- .mark{width:35px;height:6px;border-radius:4px;background:#20574d;margin:0 0 20px}.brand{font-size:18px;line-height:22px;font-weight:600;color:#527875;margin:0 0 14px}
11
- h1{font-size:42px;line-height:50px;letter-spacing:-1.3px;margin:0 0 14px;font-weight:700}.intro{font-size:20px;line-height:28px;color:#41516a;margin:0}
12
- ul{padding:0;margin:23px 0 35px;list-style:none;color:#31415a;font-size:18px;line-height:33px}li{position:relative;padding-left:33px}li:before{content:'';position:absolute;left:7px;top:14px;width:7px;height:7px;background:#2b7864;border-radius:50%}
13
- .notice{margin:0 0 16px;padding:12px 16px;border-radius:8px;background:#fdf6ec;color:#6b3d0f;font-size:16px;line-height:22px}
14
- .cf-turnstile{min-height:65px}.button{display:flex;align-items:center;justify-content:center;gap:12px;width:100%;margin-top:28px;min-height:58px;border:1px solid #86a49f;border-radius:11px;background:linear-gradient(110deg,#225b51,#1e5148 60%,#20584d);color:#fff;font:inherit;font-size:20px;font-weight:700;line-height:26px;padding:15px;text-align:center;text-decoration:none;cursor:pointer}
15
- .button:disabled{cursor:progress}[data-state=waiting] .button{background:#edf3f1;border-color:#78968f;color:#1b4d44;font-weight:600;cursor:default}
16
- .spinner{display:none;flex:none;width:20px;height:20px;border:3px solid rgba(255,255,255,.35);border-top-color:#fff;border-radius:50%;animation:spin .8s linear infinite}[data-state=working] .spinner,[data-state=signing] .spinner{display:block}@keyframes spin{to{transform:rotate(360deg)}}
17
- ol.steps{display:none;list-style:none;margin:22px auto 0;padding:0;width:max-content;max-width:100%;font-size:16px;line-height:22px;color:#50617e}[data-state=working] .steps,[data-state=signing] .steps,[data-state=slow] .steps{display:grid;gap:10px}[data-state=failed] .note{display:none}
18
- .steps li{padding-left:30px;margin:0}.steps li:before{left:0;top:2px;width:18px;height:18px;box-sizing:border-box;background:none;border:2px solid #7f8da1}.steps [data-at=done]{color:#31415a}.steps [data-at=done]:before{content:'✓';border:0;background:#2b7864;color:#fff;font-size:12px;line-height:18px;font-weight:700;text-align:center}.steps [data-at=now]{color:#192333;font-weight:600}.steps [data-at=now]:before{border-color:#c9dcd7;border-top-color:#20574d;animation:spin .8s linear infinite}
19
- .visually-hidden{position:absolute;width:1px;height:1px;margin:-1px;overflow:hidden;clip:rect(0 0 0 0);white-space:nowrap}
20
- @media(prefers-reduced-motion:reduce){.spinner,.steps [data-at=now]:before{animation:none}.steps [data-at=now]:before{border-color:#20574d}}.button:focus-visible,a:focus-visible{outline:3px solid #388c77;outline-offset:4px}
21
- .note{font-size:15px;line-height:21px;color:#50617e;text-align:center;margin:21px 0 0}.secondary{text-align:center;margin:22px 0 0;font-size:18px;line-height:24px;font-weight:600}.secondary a{color:#1b554e;text-underline-offset:3px}
22
- .back{text-align:center;margin:32px 0 0;font-size:15px;line-height:22px}.back a{color:#1b554e;text-underline-offset:3px}
23
- .ended main{width:min(560px,calc(100% - 32px));margin-top:clamp(32px,26vh,262px)}.ended .card{padding:28px 36px 42px}.ended .brand{font-size:16px;margin-bottom:26px}.ended h1{font-size:36px;line-height:44px;margin-bottom:22px}.ended .intro{margin-bottom:36px}.ended .button{margin-top:0;font-size:18px}
24
- @media(min-width:600px) and (max-width:1023px){main{width:calc(100% - 64px);max-width:666px;margin:100px auto 48px}.card{padding:36px}h1{font-size:38px;line-height:46px}.intro{font-size:18px;line-height:27px}}
25
- @media(max-width:599px){main,.ended main{width:calc(100% - 32px);margin:32px auto}.ended .card{padding:26px 22px}.ended h1{font-size:29px;line-height:36px}.ended .intro{font-size:17px;line-height:25px;margin-bottom:28px}.card{padding:26px 22px;border-radius:16px}.brand{font-size:16px}h1{font-size:34px;line-height:41px;letter-spacing:-.8px}.intro{font-size:17px;line-height:25px}ul{font-size:16px;line-height:25px;margin:22px 0 26px}li{padding-left:22px;margin-bottom:9px}li:before{left:2px;top:9px}.button{font-size:18px;margin-top:22px}.note{font-size:13px;line-height:20px}.steps{font-size:15px}.back{margin-top:24px;font-size:14px}}`;
26
-
27
10
  // The console's chunks, fetched at idle priority while the visitor waits for
28
11
  // the check and sign-in (native/warm.mjs). The browser ignores a failed one,
29
12
  // and one with prefetching turned off fetches none.
@@ -31,9 +14,9 @@ const warm = env => (env.DEMO_WARM ?? []).map(path => `<link rel="prefetch" href
31
14
 
32
15
  function html(env, { page, title, body, status = 200, headers = {}, turnstile = false, refresh = false }) {
33
16
  const nonce = crypto.randomUUID().replaceAll('-', ''), site = new URL(env.DEMO_SITE);
34
- const policy = `default-src 'none'; style-src 'nonce-${nonce}'; img-src 'self'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'${turnstile ? `; script-src 'nonce-${nonce}' https://challenges.cloudflare.com; frame-src https://challenges.cloudflare.com; connect-src 'self' https://challenges.cloudflare.com` : ''}`;
35
- return new Response(`<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="robots" content="noindex"><title>${escape(title)}</title>${refresh ? '<noscript><meta http-equiv="refresh" content="2;url=/__demo/status"></noscript>' : ''}<style nonce="${nonce}">${STYLE}</style>${turnstile ? `<noscript><style nonce="${nonce}">#note{display:none}</style></noscript><script nonce="${nonce}">${START_SCRIPT}</script><script nonce="${nonce}" src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script>${warm(env)}` : ''}</head>
36
- <body class="${page}"><main><section class="card" aria-labelledby="title"><div class="mark" aria-hidden="true"></div><p class="brand">${escape(env.DEMO_NAME)}</p>${body}</section><nav class="back" aria-label="Return"><a href="${escape(site.origin)}/">Back to ${escape(site.host)}</a></nav></main></body></html>`,
17
+ const policy = `default-src 'none'; style-src 'self' 'nonce-${nonce}'; font-src 'self'; img-src 'self'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'; script-src 'nonce-${nonce}'${turnstile ? ` https://challenges.cloudflare.com; frame-src https://challenges.cloudflare.com; connect-src 'self' https://challenges.cloudflare.com` : ''}`;
18
+ return new Response(`<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="robots" content="noindex"><title>${escape(title)}</title>${refresh ? '<noscript><meta http-equiv="refresh" content="2;url=/__demo/status"></noscript>' : ''}${(env.DEMO_THEME ?? []).map(path => `<link rel="stylesheet" href="${escape(path)}">`).join('')}<style nonce="${nonce}">${STYLE}</style>${turnstile ? `<noscript><style nonce="${nonce}">#note{display:none}</style></noscript><script nonce="${nonce}">${START_SCRIPT}</script><script nonce="${nonce}" src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script>${warm(env)}` : ''}</head>
19
+ <body class="${page}"><main><section class="card" aria-labelledby="title"><p class="brand"><img class="mark" alt="${escape(env.DEMO_NAME)}" width="40" height="40" hidden><span>${escape(env.DEMO_NAME)}</span></p>${body}</section><nav class="back" aria-label="Return"><a href="${escape(site.origin)}/">Back to ${escape(site.host)}</a></nav></main><script nonce="${nonce}">${LOGO_SCRIPT}</script></body></html>`,
37
20
  { status, headers: { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store', 'content-security-policy': policy, 'referrer-policy': 'strict-origin', 'x-content-type-options': 'nosniff', ...headers } });
38
21
  }
39
22
 
@@ -4,7 +4,7 @@ import { ownerWorkerName } from '../deploy-kit/owner.mjs';
4
4
  import { siteDeclaration } from '../deploy-kit/site.mjs';
5
5
 
6
6
  // Native Loader front, admission and visitor session support.
7
- export const DEMO_FILES = ['worker.mjs', 'capacity.mjs', 'claim.mjs', 'pages.mjs', 'start-script.mjs', 'diagnostics.mjs', 'operator.mjs', 'measurement.mjs'];
7
+ export const DEMO_FILES = ['worker.mjs', 'capacity.mjs', 'claim.mjs', 'pages.mjs', 'page-style.mjs', 'theme-script.mjs', 'start-script.mjs', 'diagnostics.mjs', 'operator.mjs', 'measurement.mjs'];
8
8
  const FRONT_SET = /^(?:RER_.*|SESSION_SECRET|SETUP_TOKEN|OWNER_EMAIL)$/;
9
9
  const SECRET = /SECRET|TOKEN|PASSWORD|_KEY$/;
10
10
  export function demoDeclaration(value, originVar) {
@@ -0,0 +1,18 @@
1
+ // Like the Lobby, load the seed's served theme and show its mark beside the
2
+ // product name. Resolve the CSS token in the browser so dark-mode marks work
3
+ // too. A missing or broken image leaves the product's text visible.
4
+ export const LOGO_SCRIPT = `(()=>{
5
+ const mark=document.querySelector('.brand img'),name=document.querySelector('.brand span');
6
+ const update=()=>{
7
+ const token=getComputedStyle(document.documentElement).getPropertyValue('--rer-logo');
8
+ const match=/url\\(\\s*(['"]?)([^'"\\)]+)\\1\\s*\\)/.exec(token);
9
+ mark.hidden=true;name.removeAttribute('aria-hidden');
10
+ if(!match)return;
11
+ const url=new URL(match[2].trim(),location.href);
12
+ if(url.origin!==location.origin)return;
13
+ mark.onload=()=>{mark.hidden=false;name.setAttribute('aria-hidden','true')};
14
+ mark.onerror=()=>{mark.hidden=true;name.removeAttribute('aria-hidden')};
15
+ mark.src=url.href;
16
+ };
17
+ update();matchMedia('(prefers-color-scheme: dark)').addEventListener('change',update);
18
+ })()`;
@@ -0,0 +1,27 @@
1
+ import { mkdir, writeFile } from 'node:fs/promises';
2
+ import { dirname, join } from 'node:path';
3
+ import { fileURLToPath } from 'node:url';
4
+ import { shared } from '../../cli/shared.mjs';
5
+
6
+ const prefix = '/__demo/theme';
7
+
8
+ // Ship the maintained default seed's actual theme, font and mark, rather than
9
+ // another set of fallback values. Namespace its assets so they cannot replace
10
+ // a seed's own files. The seed stylesheet follows it, just as in the Lobby;
11
+ // the browser resolves imports, token aliases and light/dark media rules.
12
+ export async function prepareTheme(folder, assets) {
13
+ const { loadSeedTheme } = await shared('theme-files.mjs');
14
+ const defaults = await loadSeedTheme(fileURLToPath(new URL('../../default-seed/', import.meta.url)), 'public/rer-theme.css');
15
+ const paths = [];
16
+ for (const [path, file] of defaults.files) {
17
+ const target = join(folder, 'assets', prefix, path);
18
+ await mkdir(dirname(target), { recursive: true });
19
+ const bytes = file.type === 'text/css'
20
+ ? file.bytes.toString().replace(/url\((['"]?)(\/[^)'"\s]+)\1\)/g, (_, quote, url) => `url(${quote}${prefix}${url}${quote})`)
21
+ : file.bytes;
22
+ await writeFile(target, bytes);
23
+ paths.push(prefix + path);
24
+ }
25
+ const has = path => assets.some(file => file.path === path);
26
+ return { paths, stylesheets: [prefix + '/rer-theme.css', ...(has('fonts/fonts.css') ? ['/fonts/fonts.css'] : []), ...(has('rer-theme.css') ? ['/rer-theme.css'] : [])] };
27
+ }
@@ -20,7 +20,7 @@ schema at construction. Apply the bookkeeping migration first. Policy fields:
20
20
  | --- | --- |
21
21
  | `product` | Required archive product identity. |
22
22
  | `excludeTables` | Additional exact names or prefixes ending in `%`. |
23
- | `includeTables` | Explicit identity tables to include despite default exclusions. |
23
+ | `includeTables` | Explicit identity tables to include despite default identity exclusions. Mandatory platform bookkeeping tables cannot be included. |
24
24
  | `derivedTables` | Virtual/FTS tables to regenerate; their shadow tables are excluded. |
25
25
  | `secretColumns` | Per-table nullable base columns omitted at export and written as NULL. Primary keys and generated columns cannot be secret. |
26
26
  | `exportRow(table,row)` | Pure, synchronous, idempotent transformation of encoded base cells. Restore writes its result verbatim; SQLite recomputes generated cells. |
@@ -32,14 +32,35 @@ schema at construction. Apply the bookkeeping migration first. Policy fields:
32
32
  | `extraPayloads` | Named `{produce,consume}` pairs. Producers return bytes or a byte stream. Consumers receive a verified byte stream and descriptor before D1 commit. |
33
33
  | `partBytes` | Maximum part size, default 16 MiB, configurable from 1 KiB to 16 MiB. |
34
34
 
35
- Archives always skip Base's own tables: SQLite and Cloudflare internals, the
36
- migration ledgers (`d1_migrations`, `_rer_deploy_migrations`,
37
- `_rer_portability_migrations`), portability bookkeeping, development outboxes,
38
- operator identity and dependency secrets. `excludedTables` lists them (exact
39
- names, or prefixes ending in `%`), and `excludedTable(name, policy)` says whether
40
- an archive under that policy skips a table. A seed's own table inventory, which
41
- refuses tables it does not know, uses the same predicate, so it does not refuse a
42
- table Base created and never archives.
35
+ `bookkeeping.mjs` defines the mandatory `platformTables` list. These tables
36
+ belong to the current runtime or environment and never travel:
37
+
38
+ | Tables (exact names or prefixes ending in `%`) | Reason |
39
+ | --- | --- |
40
+ | `sqlite_%`, `_cf_%` | SQLite and Cloudflare internals. |
41
+ | `d1_migrations`, `_rer_deploy_migrations`, `_rer_portability_migrations`, `rer_migrations` | Destination migration history. Current Base uses the D1 and `_rer_` ledgers; `rer_migrations` is excluded for historical compatibility. |
42
+ | `rer_portability_%` | Transfer jobs, snapshots, staging, journals and recovery bookkeeping. |
43
+ | `rer_mail_development_outbox`, `rer_ai_development_outbox` | Local development effect captures. |
44
+ | `rer_dependency_keys`, `rer_dependency_pending`, `rer_dependency_codes` | Provider secrets and transient linking credentials. |
45
+ | `rer_command_receipts` | Command replay receipts tied to the source installation. |
46
+ | `rer_erasure_blobs`, `rer_erasure_guard`, `rer_erasure_scope`, `rer_erasure_rates` | Erasure work queues, transaction guards, working scope and local rate limits. |
47
+ | `rer_demo` | Local demo setup markers. |
48
+
49
+ `excludedTables` combines this list with the default identity exclusions `oa_%`
50
+ and `oc_%`. Policy may override those identity exclusions through `includeTables`,
51
+ but naming a mandatory platform table there refuses construction with
52
+ `platform_table:<name>` and a message identifying the table.
53
+ `excludedTable(name, policy)` always returns true for mandatory platform tables,
54
+ regardless of policy, so export cannot include them. A seed's own table inventory,
55
+ which refuses tables it does not know, uses the same predicate, so it does not
56
+ refuse a table Base created and never archives.
57
+
58
+ These are specific exclusions, not a blanket exclusion of `rer_%` tables.
59
+ Durable erasure receipts in `rer_erasure_operations`, installation identity in
60
+ `rer_dependency_installation`, retained links in `rer_dependency_link`, and mail
61
+ delivery receipts and state in `rer_mail_event_receipts` and
62
+ `rer_mail_event_states` remain portable unless product policy excludes them.
63
+ Seed-owned tables are likewise discovered and remain portable by default.
43
64
 
44
65
  For native local CLI transfers, declare executable policy through
45
66
  `localPortability({database, buckets, policyModule})`. `policyModule` is a
@@ -69,9 +90,6 @@ it continues to accept serializable object policies. Malformed primary or later
69
90
  journal JSON is reported once as `archive_json` with exit 2, before local data
70
91
  migrations, without echoing archive content or parser diagnostics.
71
92
 
72
- Default exclusions are `sqlite_%`, `_cf_%`, `d1_migrations`,
73
- `rer_portability_%`, `rer_mail_development_outbox`, `oa_%`, `oc_%`,
74
- `rer_dependency_keys`, `rer_dependency_pending` and `rer_dependency_codes`.
75
93
  When a policy includes operator-auth tables, local dev identity still never
76
94
  travels, and no policy option changes that: export drops `oa_sessions` rows of
77
95
  kind `dev`, every credential recorded in `oa_dev_credentials` (and that table's
@@ -147,6 +165,15 @@ commit. Local `rer import` verifies the whole input before even applying
147
165
  destination migrations. `contentHash` remains the content-equivalence digest;
148
166
  `archiveHash` changes when export time, source metadata or part boundaries change.
149
167
 
168
+ Older version 2 archives may contain tables now classified as mandatory platform
169
+ bookkeeping. Import verifies the original archive in full, including its
170
+ manifest, every part, `contentHash` and `archiveHash`; excluded rows still count
171
+ in those integrity checks. Their rows are never staged or restored. Only these
172
+ mandatory tables are removed from the effective schema before compatibility
173
+ checks; this does not waive checks on any remaining table. `imports.status(id)`
174
+ reports their names in `skippedTables`. File and HTTP CLI imports print one line
175
+ when any are skipped: `Skipped platform bookkeeping tables: <names>.`
176
+
150
177
  Version 1 archives lack this binding and are refused with
151
178
  `legacy_archive_unverifiable` (HTTP 409; local `rer import` exit 4), before
152
179
  restoration. Upgrade the original installation and export a new archive.
@@ -444,6 +471,7 @@ modules/portability/lines.mjs
444
471
  modules/portability/hash.mjs
445
472
  modules/portability/format.mjs
446
473
  modules/portability/catalogue.mjs
474
+ modules/portability/bookkeeping.mjs
447
475
  modules/portability/storage.mjs
448
476
  modules/portability/copies.mjs
449
477
  modules/portability/dev-data.mjs
@@ -0,0 +1,25 @@
1
+ import { canonicalJson, sha256 } from './hash.mjs';
2
+
3
+ // Installation authority and transient working state, never product data.
4
+ // Keep exact names except for SQLite/Cloudflare internals and portability's
5
+ // own dynamic copies. Durable erasure receipts and dependency bindings travel.
6
+ export const platformTables=Object.freeze([
7
+ 'sqlite_%','_cf_%','d1_migrations','rer_migrations',
8
+ '_rer_deploy_migrations','_rer_portability_migrations','rer_portability_%',
9
+ 'rer_mail_development_outbox','rer_ai_development_outbox',
10
+ 'rer_dependency_keys','rer_dependency_pending','rer_dependency_codes',
11
+ 'rer_command_receipts','rer_erasure_blobs','rer_erasure_guard',
12
+ 'rer_erasure_scope','rer_erasure_rates','rer_demo',
13
+ ]);
14
+ export const identifier=value=>value.replace(/[A-Z]/g,letter=>letter.toLowerCase());
15
+ export const matches=(table,pattern)=>pattern.endsWith('%') ? identifier(table).startsWith(identifier(pattern.slice(0,-1))) : identifier(table)===identifier(pattern);
16
+ export const platformTable=table=>platformTables.some(pattern=>matches(table,pattern));
17
+ // Compatibility uses the product schema, while the original manifest and parts
18
+ // remain intact for end-to-end archive verification, including skipped rows.
19
+ export async function restorableManifest(manifest) {
20
+ const tables=manifest.tables.filter(table=>!platformTable(table.name));
21
+ if(tables.length===manifest.tables.length)return manifest;
22
+ return {...manifest,tables,schema:await sha256(canonicalJson(tables.map(({name,sql})=>({name,sql})))),
23
+ excludedColumns:Object.fromEntries(Object.entries(manifest.excludedColumns).filter(([name])=>!platformTable(name)))};
24
+ }
25
+ export const skippedTablesLine=tables=>'Skipped platform bookkeeping tables: '+tables.join(', ')+'.';
@@ -1,10 +1,8 @@
1
- import { all, insist, quote } from './storage.mjs';
1
+ import { all, insist, quote, PortabilityError } from './storage.mjs';
2
2
  import { canonicalJson, sha256, base64, unbase64, utf8 } from './hash.mjs';
3
- /** Tables no archive carries: SQLite/Cloudflare internals, migration ledgers,
4
- * portability bookkeeping, development outboxes, operator identity, dependency
5
- * secrets and refresh-safe command receipts. Exact names, or prefixes ending in `%`. Seeds' own table inventories
6
- * skip the same tables through `excludedTable`. */
7
- export const excludedTables=Object.freeze(['sqlite_%','_cf_%','d1_migrations','_rer_deploy_migrations','_rer_portability_migrations','rer_portability_%','rer_mail_development_outbox','rer_ai_development_outbox','oa_%','oc_%','rer_dependency_keys','rer_dependency_pending','rer_dependency_codes','rer_command_receipts']);
3
+ import { platformTables, platformTable, identifier, matches } from './bookkeeping.mjs';
4
+ /** Platform bookkeeping is mandatory; operator identity remains a seed choice. */
5
+ export const excludedTables=Object.freeze([...platformTables,'oa_%','oc_%']);
8
6
  const fields=['product','excludeTables','includeTables','derivedTables','secretColumns','exportRow','replaceableTables','requiredConfiguration','validateStaged','acceptedSchemas','buckets','extraPayloads','partBytes'];
9
7
  const name=value=>typeof value==='string' && value.length>0 && value.length<=512 && !/[\x00-\x1f]/.test(value);
10
8
  const names=value=>Array.isArray(value) && value.every(name) && new Set(value).size===value.length;
@@ -12,6 +10,10 @@ export function policy(input) {
12
10
  insist(input && typeof input==='object' && Object.keys(input).every(k=>fields.includes(k)) && name(input.product),'invalid_policy');
13
11
  const result={excludeTables:[],includeTables:[],derivedTables:[],secretColumns:{},replaceableTables:[],requiredConfiguration:[],acceptedSchemas:{},buckets:{},extraPayloads:{},partBytes:16*1024*1024,...input};
14
12
  for(const key of ['excludeTables','includeTables','derivedTables','replaceableTables','requiredConfiguration']) insist(names(result[key]),'invalid_policy_'+key);
13
+ for(const table of result.includeTables) if(platformTable(table)) {
14
+ const error=new PortabilityError('platform_table:'+table);
15
+ error.message='includeTables cannot include platform-owned table "'+table+'".';throw error;
16
+ }
15
17
  for(const key of ['exportRow','validateStaged']) insist(result[key]===undefined || typeof result[key]==='function','invalid_policy_'+key);
16
18
  insist(Number.isSafeInteger(result.partBytes) && result.partBytes>=1024 && result.partBytes<=16*1024*1024,'invalid_part_limit');
17
19
  for(const [table,columns] of Object.entries(result.secretColumns)) insist(name(table) && names(columns),'invalid_secret_columns');
@@ -25,17 +27,14 @@ export function policy(input) {
25
27
  for(const key of ['secretColumns','acceptedSchemas','buckets','extraPayloads'])result[key]=Object.assign(Object.create(null),result[key]);
26
28
  return {...result,excludeTables:[...excludedTables,...result.excludeTables]};
27
29
  }
28
- const identifier=value=>value.replace(/[A-Z]/g,letter=>letter.toLowerCase());
29
- const matches=(table,pattern)=>pattern.endsWith('%') ? identifier(table).startsWith(identifier(pattern.slice(0,-1))) : identifier(table)===identifier(pattern);
30
- /** True when no archive carries `table`: one of Base's `excludedTables`, or one of
31
- * the policy's own `excludeTables`, unless the policy includes it explicitly. */
32
- export const excludedTable=(table,input={})=>!(input.includeTables??[]).includes(table) && [...excludedTables,...(input.excludeTables??[])].some(pattern=>matches(table,pattern));
30
+ /** Platform tables are always excluded; explicit includes override only optional exclusions. */
31
+ export const excludedTable=(table,input={})=>platformTable(table) || (!(input.includeTables??[]).includes(table) && [...excludedTables,...(input.excludeTables??[])].some(pattern=>matches(table,pattern)));
33
32
  async function archived(db,config) {
34
33
  const objects=await all(db,"SELECT name,sql,type,tbl_name FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name");
35
34
  const kinds=await all(db,'PRAGMA table_list');
36
35
  const derived=config.derivedTables;
37
- const excluded=objects.filter(x=>x.type==='table' && !config.includeTables.includes(x.name) &&
38
- (config.excludeTables.some(p=>matches(x.name,p)) || derived.some(p=>x.name===p || (kinds.find(k=>k.name===x.name)?.type==='shadow' && x.name.startsWith(p+'_')))));
36
+ const excluded=objects.filter(x=>x.type==='table' && (excludedTable(x.name,config) || (!config.includeTables.includes(x.name) &&
37
+ derived.some(p=>x.name===p || (kinds.find(k=>k.name===x.name)?.type==='shadow' && x.name.startsWith(p+'_'))))));
39
38
  objects.sort((a,b)=>a.name<b.name?-1:a.name>b.name?1:0);
40
39
  return {objects,kinds,excluded,entries:objects.filter(x=>x.type==='table' && !excluded.some(y=>y.name===x.name))};
41
40
  }
@@ -4,6 +4,7 @@ import { Readable } from 'node:stream';
4
4
  import { pathToFileURL, fileURLToPath } from 'node:url';
5
5
  import { options, cookieFile, absent, privateFile, checkpoint, promote } from './cli-options.mjs';
6
6
  import { readArchive } from './format.mjs';
7
+ import { skippedTablesLine } from './bookkeeping.mjs';
7
8
  import { canonicalJson } from './hash.mjs';
8
9
  import { transferClient, transferDiagnostic } from './http-client.mjs';
9
10
  export async function main(args) {
@@ -37,6 +38,7 @@ export async function main(args) {
37
38
  else closing=record;
38
39
  }
39
40
  const result=await get('imports/'+id+'/finalize','POST',{contentHash:closing.contentHash,archiveHash:closing.archiveHash});
41
+ if(result.skippedTables?.length)process.stdout.write(skippedTablesLine(result.skippedTables)+'\n');
40
42
  process.stdout.write('Import '+result.phase+'.\nRequired configuration: '+result.requiredConfiguration.join(', ')+'\n');return;
41
43
  }
42
44
  let collection='exports';
@@ -69,7 +69,11 @@ export async function importArchive(service, path, transient, journal, progress)
69
69
  if (file === journal && record.kind === 'manifest') later = { sourceIdentity: record.sourceIdentity, epoch: record.epoch, journal: record.journal };
70
70
  }
71
71
  for await (const record of fileRecords(path)) {
72
- if (record.kind === 'manifest') { manifest = record; id = await service.imports.create(manifest); }
72
+ if (record.kind === 'manifest') {
73
+ manifest = record; id = await service.imports.create(manifest);
74
+ const { skippedTables } = await service.imports.status(id);
75
+ if (skippedTables?.length) progress?.({ phase: 'skipped', tables: skippedTables });
76
+ }
73
77
  else if (record.kind === 'part') {
74
78
  await service.imports.uploadPart(id, record.index, record);
75
79
  staged += Buffer.byteLength(canonicalJson(record)) + 1;
@@ -79,6 +83,6 @@ export async function importArchive(service, path, transient, journal, progress)
79
83
  }
80
84
  progress?.({ phase: 'finishing', total: manifest.partCount });
81
85
  const status = await service.imports.finalize(id, { contentHash: closing.contentHash, archiveHash: closing.archiveHash, ...(later ? { laterJournal: later } : {}) }); complete = true;
82
- return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}) };
86
+ return { ...receipt(manifest, closing, transient), ...(status.bridged ? { bridged: status.bridged } : {}), ...(status.skippedTables ? { skippedTables: status.skippedTables } : {}) };
83
87
  } finally { if (id && !complete) await service.imports.abort(id); }
84
88
  }
@@ -3,6 +3,7 @@ import { stagePart } from './import-staging.mjs';
3
3
  import { refuseProviderData } from './provider-data.mjs';
4
4
  import { refuseDevData } from './dev-data.mjs';
5
5
  import { catalogue, validateRow, bindCell, writableColumns } from './catalogue.mjs';
6
+ import { platformTable, restorableManifest } from './bookkeeping.mjs';
6
7
  import { canonicalJson, unbase64 } from './hash.mjs';
7
8
  import { checkManifest, checkPart, ContentVerifier, ArchiveVerifier } from './format.mjs';
8
9
  import { fence, mergeJournal, tombstone } from './journal.mjs';
@@ -16,7 +17,9 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
16
17
  insist(manifest.product===config.product,'wrong_product');
17
18
  // Explicit schema upgrades can reconcile rows, but cannot attest an interrupted deployment.
18
19
  await appliedMigrations(db);await state(db);
19
- const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[manifest.schema];
20
+ const original=manifest;manifest=await restorableManifest(original);
21
+ const skippedTables=original.tables.filter(table=>platformTable(table.name)).map(table=>table.name);
22
+ const cat=await catalogue(db,config),upgrade=config.acceptedSchemas[original.schema];
20
23
  const same=manifest.schema===cat.identity;
21
24
  const bridge=same || upgrade?undefined:await additiveBridge(db,manifest,cat,context.migrations);
22
25
  if(same && !upgrade)await sameSchemaMigrations(db,manifest,cat,context.migrations);
@@ -33,7 +36,7 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
33
36
  for(const o of manifest.objects) insist(buckets[o.bucket] && config.buckets[o.bucket] && o.key.startsWith(config.buckets[o.bucket].prefix??''),'bucket_inventory');
34
37
  insist(canonicalJson(manifest.payloads.map(x=>x.name))===canonicalJson(Object.keys(config.extraPayloads).sort()),'payload_inventory');
35
38
  const id=crypto.randomUUID(),copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
36
- const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest,copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {})}};
39
+ const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge?{bridge}:{}),...(legacyRecovery?{legacyRecovery}: {})}};
37
40
  try {
38
41
  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
39
42
  statement(db,'UPDATE rer_portability_state SET import_job=? WHERE id=1',id),
@@ -52,6 +55,7 @@ export async function uploadPart(context,id,index,part) {
52
55
  if(item.kind==='row') {
53
56
  const table=manifest.tables.find(x=>x.name===item.table);insist(table && Number.isSafeInteger(item.index) && item.index>=0 && item.index<table.count,'row_inventory');
54
57
  validateRow(table,item.row,manifest.excludedColumns[table.name]??[]);
58
+ if(platformTable(table.name))continue;
55
59
  const target=row.metadata.targets.find(x=>x.name===table.name);
56
60
  const data=upgrade?upgrade.upgradeRow(table.name,structuredClone(item.row)):item.row;
57
61
  insist(!(data instanceof Promise),'upgrade_row_must_be_pure');validateRow(target,data,config.secretColumns[table.name]??[]);
@@ -147,7 +151,7 @@ export async function finalizeImport(context,id,{contentHash,archiveHash,laterJo
147
151
  }
148
152
  const digest=verifier.digest();if(contentHash!==undefined)insist(contentHash===digest,'content_checksum');
149
153
  insist(archiveHash===archive.digest(),'archive_checksum',409);
150
- const checked=await fence(db,manifest,row.metadata.copies,laterJournal);
154
+ const checked=await fence(db,row.metadata.restoreManifest??manifest,row.metadata.copies,laterJournal);
151
155
  if(row.metadata.legacyRecovery)await requireLegacyEmpty(context,row.metadata.legacyRecovery);
152
156
  const {seeded,replaced}=await destinationState(context,row);
153
157
  const dependentGuards=await excludedDependentGuards(db,row.metadata.targets,replaced);
@@ -189,7 +193,7 @@ export async function finalizeImport(context,id,{contentHash,archiveHash,laterJo
189
193
  }
190
194
  export async function importStatus(context,id) {
191
195
  const row=await job(context.db,id,'import');return {id,kind:'import',phase:row.phase,completed:row.completed,total:row.metadata.manifest.partCount,
192
- contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{})};
196
+ contentHash:row.metadata.contentHash??null,requiredConfiguration:row.phase==='complete'?row.metadata.manifest.requiredConfiguration:[],...(row.metadata.bridge?{bridged:row.metadata.bridge}:{}),...(row.metadata.skippedTables?{skippedTables:row.metadata.skippedTables}:{})};
193
197
  }
194
198
  export async function abortImport(context,id) {
195
199
  const row=await job(context.db,id,'import');insist(row.phase!=='complete','import_complete',409);
@@ -28,7 +28,7 @@ export interface ArchivePolicy {
28
28
  extraPayloads?: Record<string,{produce:()=>Uint8Array|ReadableStream<Uint8Array>|Promise<Uint8Array|ReadableStream<Uint8Array>>;consume:(source:ReadableStream<Uint8Array>,descriptor:PayloadDescriptor)=>void|Promise<void>}>;
29
29
  partBytes?: number;
30
30
  }
31
- export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge}
31
+ export interface Status {id:string;kind:'export'|'import';phase:'building'|'staging'|'applying'|'complete'|'aborted';completed:number;total:number|null;/** Export only: bytes of the parts built so far. */bytes?:number;contentHash:string|null;requiredConfiguration?:string[];bridged?:Bridge;/** Platform bookkeeping ignored in an older archive. */skippedTables?:string[]}
32
32
  /** Later additive migrations an older archive was restored across. */
33
33
  export interface Bridge {fromSchema:string;toSchema:string;migrations:string[];tables:string[];columns:Record<string,string[]>}
34
34
  /** A runner's replay of the destination's applied migrations: the archive schema identity after a chosen set, and each migration's statements. */
@@ -54,7 +54,7 @@ export function makePart(index:number,items:Item[]):Promise<Part>;
54
54
  export function canonicalJson(value:unknown):string;
55
55
  export function sha256(value:string|Uint8Array):Promise<string>;
56
56
  export const MAX_LINE_BYTES:number;
57
- /** Tables no archive carries (exact names, or prefixes ending in `%`). */
57
+ /** Default exclusions (exact names, or prefixes ending in `%`); identity tables may be included. */
58
58
  export const excludedTables:readonly string[];
59
59
  /** True when no archive under this policy carries `table`. */
60
60
  export function excludedTable(table:string,policy?:Pick<ArchivePolicy,'excludeTables'|'includeTables'>):boolean;
@@ -1,3 +1,4 @@
1
+ import { skippedTablesLine } from './bookkeeping.mjs';
1
2
  // Human progress for a long export or import. The caller sends events; this turns
2
3
  // them into at most one line every few seconds, with an estimate of the time left
3
4
  // once a phase has two samples. The first and last line of every phase always show.
@@ -38,6 +39,7 @@ export function transferProgress(report, { interval = 3000, now = () => performa
38
39
  }
39
40
  };
40
41
  return event => {
42
+ if (event.phase === 'skipped') { report(skippedTablesLine(event.tables)); return; }
41
43
  if (event.phase === 'start') {
42
44
  const noun = event.direction === 'export' ? `Exporting about ${formatBytes(event.bytes)} of local data` : `Restoring a ${formatBytes(event.bytes)} archive`;
43
45
  if (event.direction === 'export') expected = event.bytes;
@@ -5,7 +5,7 @@ import { inspectSource, newOutput, git } from './source.mjs';
5
5
  import { verifySource } from './source-verification.mjs';
6
6
  import { catalogue, writeCatalogue, sha256, json } from './catalogue.mjs';
7
7
  import { publicExport, cleanHistory, annotate, releaseMessage } from './export.mjs';
8
- import { publicationScan, validatePublicationDestination } from './publication-scan.mjs';
8
+ import { publicationScan, sourcePublicationScan, validatePublicationDestination } from './publication-scan.mjs';
9
9
  import { publicDestination, requirePublicDestination, remotePublicHead, publicChild, assertPublicTree } from './public-child.mjs';
10
10
  import { writePublishHandoff } from './publish-handoff.mjs';
11
11
  import { releasePublicBase } from './public-base.mjs';
@@ -53,28 +53,9 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
53
53
  // when available, retaining the selected source commit and tag identity.
54
54
  const exported = destination ? publicChild(source, temporary, destination, publicParent, Boolean(tag))
55
55
  : clean ? cleanHistory(source, temporary, Boolean(tag)) : null;
56
- const scanRepo = exported?.repo ?? source.repo;
57
- const scanRef = exported?.commit ?? source.commit;
58
- let report = publicationScan(scanRepo, scanRef, envFile, project, repository, scanBase);
59
- if (!exported && tag && report.status === 'PASS') {
60
- // Stage the exact final annotation before any source tag or distributable
61
- // artifact exists, in a fresh clone of the selected commit: verification
62
- // left build output in its checkout, which is not release source. Source
63
- // history was scanned above, including refs a checkout clone leaves out.
64
- const candidate = resolve(temporary, 'annotation');
65
- git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
66
- git(candidate, 'checkout', '--detach', source.commit);
67
- const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
68
- annotate(candidate, source, source.commit, staged ? `${source.tag}-publication-candidate` : source.tag);
69
- const annotations = publicationScan(candidate, source.commit, envFile, project, repository, scanBase);
70
- report.annotationInspection = annotations;
71
- if (annotations.status !== 'PASS') {
72
- report.status = annotations.status;
73
- report.complete = annotations.complete;
74
- report.guarantees.historyScanned = annotations.complete;
75
- report.findings.push(...annotations.findings);
76
- }
77
- }
56
+ let report = exported
57
+ ? publicationScan(exported.repo, exported.commit, envFile, project, repository, scanBase)
58
+ : sourcePublicationScan(source, temporary, envFile, project, repository, scanBase, Boolean(tag));
78
59
  if (destination) {
79
60
  assertPublicTree(exported.repo, exported.commit, source.tree);
80
61
  if (report.status === 'PASS' && (report.complete !== true || report.guarantees?.historyScanned !== true ||
@@ -1,5 +1,9 @@
1
1
  import { execFileSync } from 'node:child_process';
2
2
  import { resolve } from 'node:path';
3
+ import { writeFileSync } from 'node:fs';
4
+ import { git } from './source.mjs';
5
+ import { annotate } from './export.mjs';
6
+ import { runningVersion } from '../../cli/version.mjs';
3
7
 
4
8
  export function validatePublicationDestination(source, project, repository) {
5
9
  try {
@@ -9,14 +13,42 @@ export function validatePublicationDestination(source, project, repository) {
9
13
  } catch { throw Error('Publication destination must agree with the committed edition and project identities'); }
10
14
  }
11
15
 
12
- export function publicationScan(repo, ref, envFile, project, repository, publicBase) {
16
+ export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
17
+ const started = performance.now();
18
+ let report;
13
19
  try {
14
- return JSON.parse(execFileSync(process.execPath,
20
+ report = JSON.parse(execFileSync(process.execPath,
15
21
  [resolve(import.meta.dirname, '../../scripts/publish-check.mjs'), '--repo', repo, '--ref', ref,
16
- ...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : [])],
22
+ ...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : []),
23
+ ...(annotationFile ? ['--annotation-file', annotationFile] : [])],
17
24
  { stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 128 * 1024 * 1024 }));
18
25
  } catch (error) {
19
- try { return JSON.parse(error.stdout); }
26
+ try { report = JSON.parse(error.stdout); }
20
27
  catch { throw Error('Publication inspection did not produce a report'); }
21
28
  }
29
+ report.invocation = { caller: 'release', scans: 1, stagedAnnotation: Boolean(annotationFile), version: runningVersion(),
30
+ seconds: (performance.now() - started) / 1000 };
31
+ return report;
32
+ }
33
+
34
+ // This is the ordinary release gate, also usable without installing/building a
35
+ // seed. The scanner sees the original checkout and every source ref; only the
36
+ // new annotation is read from the clean candidate, never verification output.
37
+ export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged) {
38
+ const started = performance.now();
39
+ let annotationFile;
40
+ if (tagged) {
41
+ const candidate = resolve(temporary, 'annotation');
42
+ git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
43
+ git(candidate, 'checkout', '--detach', source.commit);
44
+ const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
45
+ const name = staged ? `${source.tag}-publication-candidate` : source.tag;
46
+ annotate(candidate, source, source.commit, name);
47
+ annotationFile = resolve(temporary, 'annotation.tag');
48
+ writeFileSync(annotationFile, git(candidate, 'cat-file', 'tag', `refs/tags/${name}`), { mode: 0o600 });
49
+ }
50
+ const stagingSeconds = (performance.now() - started) / 1000;
51
+ const report = publicationScan(source.repo, source.commit, envFile, project, repository, publicBase, annotationFile);
52
+ Object.assign(report.invocation, { stagingSeconds, totalSeconds: (performance.now() - started) / 1000 });
53
+ return report;
22
54
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.58",
3
+ "version": "0.2.60",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -4,6 +4,20 @@ See [setup, command and scope](../publish-check-README.md).
4
4
 
5
5
  `git_history.py` delegates revision resolution, traversal, tree inventory and
6
6
  blob reads to system Git (`rev-parse`, `rev-list`, `ls-tree`, `cat-file --batch`).
7
+ Before any tree inventory asks for blob sizes, `history_hydration.py` counts
8
+ missing objects and hydrates a partial clone with one `fetch --refetch --no-filter`
9
+ from its promisor remote. A failed fetch or remaining missing object makes the
10
+ inspection incomplete. `history.hydration` reports the missing counts, fetch
11
+ count and elapsed time; subsequent history reads disable lazy fetching.
12
+
13
+ Reports include implementation version, Python version and a digest of the
14
+ Python scanner source, plus total and per-phase wall times and call counts.
15
+ The release gate uses this same scanner entry point. When comparing it with a
16
+ standalone run, compare implementation digests as well as source trees:
17
+ `rer` inside a checkout runs the rer version that checkout pins, even when a
18
+ newer launcher started it. A tagged exact-source release stages its actual Git tag object in an isolated
19
+ clone and includes its message in the same full source scan. `annotationInspection`
20
+ is the staged annotation subset of that result; no second history walk is needed.
7
21
  `gitleaks.py` runs the pinned scanner in git, directory and stdin modes and keeps
8
22
  its report private until translating it into the value-free v3 envelope.
9
23
  Hexadecimal credential assignments also use a Gitleaks rule without generic
@@ -20,12 +20,12 @@ def main():
20
20
  parser = SafeParser(description='Scan working files and full history with system Git and pinned Gitleaks.')
21
21
  parser.add_argument('--repo', required=True)
22
22
  parser.add_argument('--ref', default='HEAD')
23
- for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir'):
23
+ for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir', 'annotation-file'):
24
24
  parser.add_argument('--' + option)
25
25
  parser.add_argument('--timeout-seconds', type=seconds, default=300)
26
26
  args = parser.parse_args()
27
27
  report = scan(args.repo, args.ref, args.public_base, args.env_file, args.project, args.repository,
28
- cache_dir=args.cache_dir, timeout=args.timeout_seconds)
28
+ cache_dir=args.cache_dir, timeout=args.timeout_seconds, annotation_file=args.annotation_file)
29
29
  print(json.dumps(report))
30
30
  return {'PASS': 0, 'FAIL': 1, 'INCOMPLETE': 2}[report['status']]
31
31
 
@@ -39,6 +39,9 @@ class History:
39
39
  raise PublicBaseError('public-base.fetch-failed') from None
40
40
  public_revisions = [self.public_commit, *sorted(self.public_tags)] if public_base else []
41
41
  self.revisions = [self.commit, *public_revisions] if public_base else ['--all', self.commit]
42
+ from history_hydration import prepare
43
+ self.hydration = prepare(self)
44
+ self.env['GIT_NO_LAZY_FETCH'] = '1'
42
45
  self.commits = self.run('rev-list', *self.revisions).decode().splitlines()
43
46
  self.public_commits = set(self.run('rev-list', *public_revisions).decode().splitlines()) if public_base else set()
44
47
  self.trees = {}
@@ -41,7 +41,7 @@ def inspect(binary, mode, repo=None, revisions=None, data=None, timeout=300, red
41
41
  if repo is not None:
42
42
  args.append(repo)
43
43
  env = {k: v for k, v in os.environ.items() if not k.startswith(('GITLEAKS_', 'GIT_'))}
44
- env.update(GIT_NO_REPLACE_OBJECTS='1', GIT_TERMINAL_PROMPT='0')
44
+ env.update(GIT_NO_REPLACE_OBJECTS='1', GIT_TERMINAL_PROMPT='0', GIT_NO_LAZY_FETCH='1')
45
45
  try:
46
46
  result = subprocess.run(args, input=data, capture_output=True, env=env, timeout=timeout)
47
47
  if result.returncode not in (0, 1):
@@ -0,0 +1,28 @@
1
+ """Fetch missing promised history in one pack before inspecting any blobs."""
2
+ from time import monotonic
3
+ from git_history import InspectionError
4
+
5
+
6
+ def prepare(history):
7
+ started = monotonic()
8
+ def missing():
9
+ return sum(line.startswith(b'?') for line in
10
+ history.run('rev-list', '--objects', '--missing=print', *history.revisions).splitlines())
11
+ before = missing()
12
+ fetches = 0
13
+ if before:
14
+ # A blob:none clone already owns the commits and trees. ls-tree -l and
15
+ # cat-file would otherwise ask its promisor for one blob at a time.
16
+ remotes = history.run('config', '--get-regexp', r'^remote\..*\.promisor$').decode().splitlines()
17
+ names = [line.rsplit(None, 1)[0][len('remote.'):-len('.promisor')]
18
+ for line in remotes if line.rsplit(None, 1)[-1].lower() == 'true']
19
+ if len(names) != 1:
20
+ raise InspectionError('history.promisor-ambiguous')
21
+ history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
22
+ '--no-recurse-submodules', '--no-write-fetch-head', '--', names[0])
23
+ fetches = 1
24
+ after = missing() if before else 0
25
+ if after:
26
+ raise InspectionError('history.missing-blob')
27
+ return dict(missingBefore=before, missingAfter=after, fetches=fetches,
28
+ seconds=round(monotonic() - started, 6))
@@ -1,5 +1,7 @@
1
1
  """Scan current files and full history using native Git and pinned Gitleaks."""
2
2
  from pathlib import Path
3
+ from scan_timings import Timings
4
+ import staged_annotation
3
5
  from git_history import History, InspectionError
4
6
  from public_base import PublicBaseError
5
7
  from gitleaks import executable, inspect as secrets, rule as secret_rule
@@ -16,15 +18,18 @@ from content_identities import ContentIdentities, candidate as identity_candidat
16
18
  from checksum_maps import ChecksumMaps
17
19
 
18
20
 
19
- def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300):
21
+ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300, annotation_file=None):
20
22
  report = new_report(public_base is not None)
21
- history = None
23
+ timings = Timings(report)
24
+ history = staged = None
22
25
  add = Findings(report, {}, frozenset()).add
23
26
  try:
24
27
  hashes = load_hashes(env_file)
25
28
  report['guarantees']['environmentChecked'] = env_file is not None
26
29
  binary = executable()
27
- history = History(repo, ref, public_base)
30
+ history = timings.call('history.prepare', History, repo, ref, public_base)
31
+ report['history']['hydration'] = history.hydration
32
+ staged = staged_annotation.load(annotation_file, history.commit)
28
33
  report['source'] = dict(commit=history.commit, tree=history.commit_tree(history.commit).decode())
29
34
  report['publicBase'] = dict(ref=history.public_label, commit=history.public_commit) if public_base else None
30
35
  config = load_config(history.selected(CONFIG))
@@ -54,7 +59,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
54
59
  tree = history.commit_tree(commit).decode()
55
60
  seen_trees.add(tree)
56
61
  scope = 'release-tree' if commit == history.commit else 'pre-existing' if commit in history.public_commits else 'newly-public'
57
- config_tree, invalid = context(history, commit, config)
62
+ config_tree, invalid = timings.call('history.context', context, history, commit, config)
58
63
  for path in invalid:
59
64
  add(INVALID if (project is not None or repository is not None) and path == 'edition.json' else 'scan.identity-invalid', path, commit, scope=scope)
60
65
  license_entry = entries_by_commit[commit].get('LICENSE')
@@ -76,7 +81,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
76
81
  local = dependencies[pair] & config_tree['localPaths']
77
82
  key = (path, oid, signature, local)
78
83
  if key not in inspected:
79
- inspected[key] = text_rules(history.blob(oid), path, config_tree, hashes)
84
+ inspected[key] = timings.call('history.text', text_rules, history.blob(oid), path, config_tree, hashes)
80
85
  rules, provenance, references = inspected[key]
81
86
  entry_scope = 'pre-existing' if (scope != 'release-tree' and oid in public_blobs) or public_paths.get(path) == oid else scope
82
87
  for rule in rules:
@@ -86,7 +91,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
86
91
  if provenance:
87
92
  add('provenance.declared-source', path, commit, oid, entry_scope)
88
93
  report['history'].update(commits=len(history.commits), trees=len(seen_trees), blobs=len(seen_blobs))
89
- selected_context, _ = context(history, history.commit, config)
94
+ selected_context, _ = timings.call('working-tree.context', context, history, history.commit, config)
90
95
  working_paths = []
91
96
  working_identities = {}
92
97
  for path, data in working_files(Path(history.repo)):
@@ -98,7 +103,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
98
103
  # their known immutable identity; new bytes have no history identity.
99
104
  entry = entries_by_commit[history.commit].get(path)
100
105
  oid = entry['oid'] if entry and entry['kind'] == 'blob' and history.blob(entry['oid']) == data else None
101
- rules, _, references = text_rules(data, path, selected_context, hashes)
106
+ rules, _, references = timings.call('working-tree.text', text_rules, data, path, selected_context, hashes)
102
107
  public_oid = public_paths.get(path)
103
108
  scope = 'pre-existing' if public_oid and (public_oid == oid or history.blob(public_oid) == data) else 'working-tree'
104
109
  for rule in rules:
@@ -110,11 +115,11 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
110
115
  add(rule, path, history.commit, oid, 'working-tree')
111
116
  if not (Path(history.repo) / 'LICENSE').is_file():
112
117
  add('license.missing', 'LICENSE', scope='working-tree')
113
- identities = {'working-tree': ContentIdentities(working_identities,
118
+ identities = {'working-tree': timings.call('identities.prepare', ContentIdentities, working_identities,
114
119
  lambda path: working_regular(Path(history.repo), path))}
115
120
  checksums = {}
116
121
  for mode in ('git', 'dir'):
117
- for hit in secrets(binary, mode, history.repo, history.revisions, timeout=timeout, redact=False):
122
+ for hit in timings.call('gitleaks.' + mode, secrets, binary, mode, history.repo, history.revisions, timeout=timeout, redact=False):
118
123
  path, commit = hit.get('File', '.'), hit.get('Commit') or history.commit
119
124
  if mode == 'dir' and Path(path).is_absolute():
120
125
  path = Path(path).resolve().relative_to(Path(history.repo).resolve()).as_posix()
@@ -133,12 +138,12 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
133
138
  entry = scoped_entries.get(path)
134
139
  return (history.blob(entry['oid']), entry['mode']) if entry and entry['kind'] == 'blob' and entry['mode'] in ('100644', '100755') else None
135
140
  if identity_scope not in checksums:
136
- checksums[identity_scope] = ChecksumMaps(
141
+ checksums[identity_scope] = timings.call('checksums.prepare', ChecksumMaps,
137
142
  working_paths if mode == 'dir' else scoped_entries,
138
143
  (lambda path: working_regular(Path(history.repo), path)) if mode == 'dir' else read_regular)
139
144
  if identity_candidate(path):
140
145
  if identity_scope not in identities:
141
- identities[identity_scope] = ContentIdentities({
146
+ identities[identity_scope] = timings.call('identities.prepare', ContentIdentities, {
142
147
  p: history.blob(e['oid']) for p, e in scoped_entries.items()
143
148
  if identity_candidate(p) and e['kind'] == 'blob' and e['mode'] in ('100644', '100755')}, read_regular)
144
149
  if identities[identity_scope].matches(path, hit):
@@ -162,10 +167,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
162
167
  scope = 'working-tree' if path in working_paths else 'pre-existing' if commit in history.public_commits else 'newly-public'
163
168
  lines.append((count, count + length, path, commit, scope))
164
169
  count += length
165
- for message_type, commit, data in history.messages():
170
+ for message_type, commit, data in staged_annotation.messages(history, staged):
166
171
  path = '@' + message_type + '-message'
167
172
  report['history']['tagMessages' if message_type == 'tag' else 'commitMessages'] += 1
168
- rules, _, references = text_rules(data, path, selected_context, hashes,
173
+ rules, _, references = timings.call('messages.text', text_rules, data, path, selected_context, hashes,
169
174
  commit_message=message_type == 'commit')
170
175
  scope = 'pre-existing' if commit in history.public_commits or commit in history.public_tags else 'release-tree' if commit == history.commit else 'newly-public'
171
176
  for rule in rules:
@@ -176,11 +181,12 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
176
181
  length = (data + b'\n').count(b'\n')
177
182
  lines.append((count, count + length, path, commit, scope))
178
183
  count += length
179
- for hit in secrets(binary, 'stdin', data=b''.join(messages), timeout=timeout):
184
+ for hit in timings.call('gitleaks.messages', secrets, binary, 'stdin', data=b''.join(messages), timeout=timeout):
180
185
  line = hit.get('StartLine', 0)
181
186
  location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
182
187
  add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
183
- states = GitHubResolver().resolve_all({repository for references, *_ in deferred for repository, _, _ in references})
188
+ states = timings.call('github.references', GitHubResolver().resolve_all,
189
+ {repository for references, *_ in deferred for repository, _, _ in references})
184
190
  for references, path, commit, oid, scope in deferred:
185
191
  for rule in reference_rules(references, states):
186
192
  add(rule, path, commit, oid, scope)
@@ -197,4 +203,7 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
197
203
  finally:
198
204
  if history:
199
205
  history.close()
200
- return finish(report)
206
+ timings.finish()
207
+ finish(report)
208
+ staged_annotation.summary(report, staged)
209
+ return report
@@ -0,0 +1,46 @@
1
+ """Numeric scan costs and implementation identity, without source file values."""
2
+ import hashlib
3
+ import json
4
+ from pathlib import Path
5
+ import platform
6
+ from time import monotonic
7
+
8
+
9
+ def implementation():
10
+ directory = Path(__file__).parent
11
+ digest = hashlib.sha256()
12
+ for path in sorted(directory.glob('*.py')):
13
+ if path.name.startswith(('test', 'coverage_')):
14
+ continue
15
+ digest.update(path.name.encode() + b'\0' + path.read_bytes())
16
+ root = directory.parent.parent
17
+ for path, key in ((root / 'inventory.json', 'cliVersion'),
18
+ (root / 'package.json', 'version'), (root / 'global/package.json', 'version')):
19
+ if path.is_file():
20
+ version = json.loads(path.read_text()).get(key)
21
+ if version:
22
+ break
23
+ else:
24
+ version = None
25
+ return dict(version=version, sha256=digest.hexdigest(), python=platform.python_version())
26
+
27
+
28
+ class Timings:
29
+ def __init__(self, report):
30
+ self.started = monotonic()
31
+ self.values = report['timings'] = dict(totalSeconds=0, phases={})
32
+ report['implementation'] = implementation()
33
+
34
+ def call(self, name, function, *args, **kwargs):
35
+ started = monotonic()
36
+ try:
37
+ return function(*args, **kwargs)
38
+ finally:
39
+ phase = self.values['phases'].setdefault(name, dict(calls=0, seconds=0))
40
+ phase['calls'] += 1
41
+ phase['seconds'] += monotonic() - started
42
+
43
+ def finish(self):
44
+ self.values['totalSeconds'] = round(monotonic() - self.started, 6)
45
+ for phase in self.values['phases'].values():
46
+ phase['seconds'] = round(phase['seconds'], 6)
@@ -0,0 +1,39 @@
1
+ """Include the actual staged Git tag in the source scan's message policy."""
2
+ import hashlib
3
+ from pathlib import Path
4
+ from git_history import InspectionError
5
+ from report import new_report, finish
6
+
7
+
8
+ def load(path, commit):
9
+ if path is None:
10
+ return None
11
+ raw = Path(path).read_bytes()
12
+ header, separator, message = raw.partition(b'\n\n')
13
+ if not separator or not header.startswith(('object ' + commit + '\ntype commit\n').encode()):
14
+ raise InspectionError('scan.annotation-invalid')
15
+ digest = hashlib.sha1 if len(commit) == 40 else hashlib.sha256
16
+ oid = digest(b'tag ' + str(len(raw)).encode() + b'\0' + raw).hexdigest()
17
+ return dict(oid=oid, message=message)
18
+
19
+
20
+ def messages(history, staged):
21
+ yield from history.messages()
22
+ if staged:
23
+ yield 'tag', staged['oid'], staged['message']
24
+
25
+
26
+ def summary(report, staged):
27
+ if not staged:
28
+ return
29
+ annotation = new_report(report.get('publicBase') is not None)
30
+ annotation.update(source=report['source'], publicBase=report.get('publicBase'),
31
+ implementation=report['implementation'])
32
+ annotation['history'].update(scope='staged-annotation', tagMessages=1)
33
+ annotation['guarantees']['environmentChecked'] = report['guarantees']['environmentChecked']
34
+ for bucket in ('findings', 'preExisting', 'warnings', 'fixtures', 'reviewed', 'info'):
35
+ annotation[bucket] = [{**item, 'commit': staged['oid'], 'commits': [staged['oid']]}
36
+ for item in report[bucket] if staged['oid'] in item['commits']]
37
+ if not report['complete']:
38
+ annotation['findings'].append(dict(rule='scan.source-incomplete', path='.'))
39
+ report['annotationInspection'] = finish(annotation)
@@ -3,6 +3,7 @@ import { spawn } from 'node:child_process';
3
3
  import { mkdtempSync, readFileSync, openSync, closeSync, rmSync } from 'node:fs';
4
4
  import { tmpdir } from 'node:os';
5
5
  import { join } from 'node:path';
6
+ import { runningVersion } from '../cli/version.mjs';
6
7
 
7
8
  const args = process.argv.slice(2);
8
9
  let seconds = 300;
@@ -54,5 +55,7 @@ if (reason) report = {
54
55
  history: { commits: 0, trees: 0, blobs: 0 },
55
56
  guarantees: { historyScanned: false, visualRecognition: false, environmentChecked: false },
56
57
  };
58
+ report.implementation ??= {};
59
+ report.implementation.version = runningVersion();
57
60
  process.stdout.write(JSON.stringify(report) + '\n');
58
61
  process.exitCode = code;