@rehearsal-db/core 0.1.0-beta.4 → 0.1.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -1
- package/README.md +11 -2
- package/SUPPORT.md +5 -0
- package/docs/commands.md +15 -4
- package/docs/getting-started.md +12 -4
- package/docs/sanitization.md +8 -5
- package/package.json +2 -2
- package/scripts/lib/rehearsal/baseline_preparation.mjs +138 -23
- package/scripts/lib/rehearsal/diagnostics.mjs +1 -1
- package/scripts/lib/rehearsal/input_discovery.mjs +155 -0
- package/scripts/lib/rehearsal/policy_review.mjs +45 -5
- package/scripts/lib/rehearsal/support_report.mjs +102 -0
- package/scripts/operations/rehearsal/rehearsal_cli.mjs +393 -128
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,29 @@ a Changelog, and versions will follow Semantic Versioning after the package exis
|
|
|
5
5
|
|
|
6
6
|
## Unreleased
|
|
7
7
|
|
|
8
|
+
## [0.1.0-beta.5] - 2026-10-01
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Scalable policy review with explicit table-level safe defaults, suggested structural
|
|
13
|
+
exceptions, per-table summaries, and real-PTY coverage of the bulk-review journey.
|
|
14
|
+
- Bounded project-local discovery for baseline records, migration ledgers, and optional
|
|
15
|
+
Storage manifests, plus a value-free structural preflight before activation.
|
|
16
|
+
- A guided `Get help` action and scriptable `rehearsal support` report with tool
|
|
17
|
+
versions, readiness statuses, privacy guarantees, and a direct bug-report link.
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- Guided policy review can classify ordinary columns in bulk as synthetic replacements
|
|
22
|
+
while keeping likely identifiers, relationships, and timestamps selected for
|
|
23
|
+
individual review. The saved policy still records every required column decision.
|
|
24
|
+
- The baseline guide offers detected inputs instead of requiring memorized paths,
|
|
25
|
+
validates referenced Storage files up front, previews counts without row values, and
|
|
26
|
+
returns to the guide with an actionable message when validation fails.
|
|
27
|
+
- Beta support no longer requires users to assemble environment details by hand; the
|
|
28
|
+
generated report omits project paths, row values, credentials, migration SQL, and
|
|
29
|
+
baseline identifiers and remains available before setup is complete.
|
|
30
|
+
|
|
8
31
|
## [0.1.0-beta.4] - 2026-10-01
|
|
9
32
|
|
|
10
33
|
### Added
|
|
@@ -118,7 +141,8 @@ a Changelog, and versions will follow Semantic Versioning after the package exis
|
|
|
118
141
|
publication uses short-lived trusted OIDC, and every release tag must already exist on
|
|
119
142
|
protected `main`.
|
|
120
143
|
|
|
121
|
-
[Unreleased]: https://github.com/Ddupasquier/rehearsal-db/compare/v0.1.0-beta.
|
|
144
|
+
[Unreleased]: https://github.com/Ddupasquier/rehearsal-db/compare/v0.1.0-beta.5...HEAD
|
|
145
|
+
[0.1.0-beta.5]: https://github.com/Ddupasquier/rehearsal-db/compare/v0.1.0-beta.4...v0.1.0-beta.5
|
|
122
146
|
[0.1.0-beta.4]: https://github.com/Ddupasquier/rehearsal-db/compare/v0.1.0-beta.3...v0.1.0-beta.4
|
|
123
147
|
[0.1.0-beta.3]: https://github.com/Ddupasquier/rehearsal-db/compare/v0.1.0-beta.2...v0.1.0-beta.3
|
|
124
148
|
[0.1.0-beta.2]: https://github.com/Ddupasquier/rehearsal-db/compare/v0.1.0-beta.1...v0.1.0-beta.2
|
package/README.md
CHANGED
|
@@ -93,6 +93,7 @@ npx rehearsal baseline prepare --records=<safe.ndjson> --ledger=<ledger.json>
|
|
|
93
93
|
npx rehearsal baseline prepare --records=<safe.ndjson> --ledger=<ledger.json> --write
|
|
94
94
|
npx rehearsal baseline create --records=<safe.ndjson> --ledger=<ledger.json>
|
|
95
95
|
npx rehearsal doctor
|
|
96
|
+
npx rehearsal support
|
|
96
97
|
npx rehearsal explain
|
|
97
98
|
npx rehearsal run --dry-run
|
|
98
99
|
npx rehearsal candidates
|
|
@@ -110,8 +111,16 @@ npx rehearsal discard
|
|
|
110
111
|
Running `npx rehearsal` in a terminal opens a state-aware guide that shows completed
|
|
111
112
|
setup steps and recommends available actions. The guide stays open after each action,
|
|
112
113
|
re-inspects the project, and advances to the next useful step. It includes an interactive
|
|
113
|
-
|
|
114
|
-
|
|
114
|
+
policy reviewer with table-level safe defaults and exception-only column review, concise
|
|
115
|
+
completion receipts, optional technical details, and bounded discovery of likely local
|
|
116
|
+
baseline inputs. Before activation it validates records, migration evidence, and optional
|
|
117
|
+
Storage files, then shows a value-free count summary for confirmation. The explicit
|
|
118
|
+
commands remain the stable interface for automation and CI.
|
|
119
|
+
|
|
120
|
+
Choose **Get help** at any stage, or run `npx rehearsal support`, to create a copy-ready
|
|
121
|
+
environment and readiness report for a GitHub issue. It works before setup and omits row
|
|
122
|
+
values, credentials, project paths, migration SQL, and baseline identifiers. Review every
|
|
123
|
+
report before sharing it.
|
|
115
124
|
|
|
116
125
|
`setup` previews a conservative first-run scaffold: the Rehearsal configuration, a
|
|
117
126
|
dedicated local-only Supabase configuration on an available port block, and protective
|
package/SUPPORT.md
CHANGED
|
@@ -6,6 +6,11 @@ Docker engine, the failing command, safe diagnostics, and the smallest reproduct
|
|
|
6
6
|
The first beta does not yet maintain a separate public discussion or feature-request
|
|
7
7
|
channel.
|
|
8
8
|
|
|
9
|
+
Run `npx rehearsal support` or choose **Get help** in the guide to collect the environment
|
|
10
|
+
and readiness portion without assembling it by hand. The report intentionally excludes
|
|
11
|
+
project paths, row values, credentials, migration SQL, and baseline identifiers. Review
|
|
12
|
+
the report before pasting it into an issue.
|
|
13
|
+
|
|
9
14
|
Do not include credentials, connection strings, source rows, baseline artifacts, private
|
|
10
15
|
URLs, or proprietary migrations. Report security problems through private vulnerability
|
|
11
16
|
reporting as described in [SECURITY.md](SECURITY.md).
|
package/docs/commands.md
CHANGED
|
@@ -15,6 +15,7 @@ values or credentials.
|
|
|
15
15
|
| `rehearsal baseline prepare --records= --ledger= --write` | Policy only | Write the draft without exposing row values. |
|
|
16
16
|
| `rehearsal baseline create --records=<path> --ledger=<path>` | Artifact only | Activate a baseline from explicit safe local inputs. |
|
|
17
17
|
| `rehearsal doctor` | No | Check dependencies, inputs, and safety barriers. |
|
|
18
|
+
| `rehearsal support` | No | Print a privacy-safe, copy-ready support report. |
|
|
18
19
|
| `rehearsal explain` | No | Print the immutable execution plan. |
|
|
19
20
|
| `rehearsal run --dry-run` | No | Alias the same plan used by `explain`. |
|
|
20
21
|
| `rehearsal candidates` | No | Print pending migrations and their exact digest. |
|
|
@@ -37,14 +38,18 @@ the digest and invalidates either form of confirmation.
|
|
|
37
38
|
`baseline create` never extracts data. The NDJSON and migration-ledger files must already
|
|
38
39
|
exist inside the project and be safe to retain. Add `--assets=<manifest.json>` to include
|
|
39
40
|
bounded local Storage bytes; every manifest `file` must also remain inside the project.
|
|
40
|
-
The guided home screen
|
|
41
|
-
|
|
41
|
+
The guided home screen scans a bounded set of small project-local JSON and NDJSON files,
|
|
42
|
+
skips generated and private runtime directories, and offers structurally matching paths
|
|
43
|
+
so they do not need to be memorized. It never displays row values. Before activation it
|
|
44
|
+
validates the selected files and referenced assets, shows row, table, migration, and asset
|
|
45
|
+
counts, and asks for confirmation. Invalid input returns to the guide without writing.
|
|
42
46
|
|
|
43
47
|
The guide is a persistent session: after setup, policy review, baseline creation, or a
|
|
44
48
|
runtime action it re-reads project state and offers the next relevant action. Generated
|
|
45
49
|
policy drafts can be completed interactively without editing JSON. Each column still
|
|
46
|
-
|
|
47
|
-
|
|
50
|
+
receives explicit action, generated, identity, and foreign-key decisions. A human may
|
|
51
|
+
apply the displayed safe preset to a table, then review only selected exceptions; the
|
|
52
|
+
guide never silently applies a preset.
|
|
48
53
|
|
|
49
54
|
`baseline prepare` reads only table and column names from the NDJSON records; row values
|
|
50
55
|
are never included in its result. Its generated policy deliberately marks every column
|
|
@@ -64,3 +69,9 @@ configuration with hosted access and optional networked services disabled. It do
|
|
|
64
69
|
overwrite an existing Rehearsal config, dedicated Supabase config, or concurrently
|
|
65
70
|
changed `.gitignore`. After writing, it includes a Doctor readiness summary. `init`
|
|
66
71
|
remains available for config-only/manual onboarding.
|
|
72
|
+
|
|
73
|
+
`support` reports the Rehearsal, Node.js, npm, Supabase CLI, Docker client, and Docker
|
|
74
|
+
server versions plus readiness check statuses. It can run before configuration exists.
|
|
75
|
+
The report deliberately excludes project names and paths, row values, credentials,
|
|
76
|
+
migration SQL, baseline identifiers, and raw command output. It reports only the count
|
|
77
|
+
of quarantined hosted variables, never their names. Always review it before sharing.
|
package/docs/getting-started.md
CHANGED
|
@@ -105,10 +105,18 @@ npx rehearsal baseline prepare \
|
|
|
105
105
|
--write
|
|
106
106
|
```
|
|
107
107
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
108
|
+
Running `npx rehearsal` instead discovers structurally matching project-local inputs and
|
|
109
|
+
offers them in the guide. Discovery is bounded, ignores generated/runtime directories,
|
|
110
|
+
and returns paths only. Before a policy draft or immutable baseline is written, Rehearsal
|
|
111
|
+
validates the selected files and presents a value-free shape and count summary.
|
|
112
|
+
|
|
113
|
+
After creating the draft, choose **Review the script** in the guide. For each table,
|
|
114
|
+
choose between reviewing every column or applying the displayed safe defaults and
|
|
115
|
+
reviewing only exceptions. Rehearsal suggests identifiers, relationship columns, and
|
|
116
|
+
timestamps as exceptions. Every saved column still records sanitization action,
|
|
117
|
+
generated status, identity status, and optional foreign-key metadata. The complete
|
|
118
|
+
policy is validated before writing, and the original draft is replaced only if it did
|
|
119
|
+
not change during review.
|
|
112
120
|
|
|
113
121
|
For noninteractive workflows, review every `REVIEW REQUIRED` field directly and remove
|
|
114
122
|
`"draft": true` only after that review. Rehearsal refuses to activate a draft.
|
package/docs/sanitization.md
CHANGED
|
@@ -19,11 +19,14 @@ whether it is an identity column, and its foreign-key target or explicit absence
|
|
|
19
19
|
draft uses `REVIEW REQUIRED` placeholders and cannot be activated until they are
|
|
20
20
|
replaced and the `draft` marker is removed.
|
|
21
21
|
|
|
22
|
-
In an interactive terminal, the guided **Review the script** action
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
22
|
+
In an interactive terminal, the guided **Review the script** action works table by table.
|
|
23
|
+
For larger tables, a human can explicitly apply safe defaults (`REPLACE`, `NEVER`
|
|
24
|
+
generated, `NO` identity, and no foreign key), then review only exceptions. Likely
|
|
25
|
+
structural columns such as `id`, `*_id`, and timestamps start selected as exceptions.
|
|
26
|
+
Users can instead review every column individually. Retaining a value is explicitly
|
|
27
|
+
labeled as sensitive, every saved field remains classified, and no preset is silently
|
|
28
|
+
applied. The reviewer validates the completed policy and refuses to overwrite a draft
|
|
29
|
+
changed during the session.
|
|
27
30
|
|
|
28
31
|
The package exports `validateSanitizationCoverage` and
|
|
29
32
|
`applySanitizationAction` as generic primitives:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rehearsal-db/core",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.5",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Safely rehearse Supabase migrations against sanitized, production-shaped PostgreSQL data.",
|
|
6
6
|
"repository": {
|
|
@@ -75,6 +75,6 @@
|
|
|
75
75
|
"devDependencies": {
|
|
76
76
|
"@lydell/node-pty": "^1.2.0-beta.15",
|
|
77
77
|
"prettier": "^3.9.6",
|
|
78
|
-
"vitest": "^
|
|
78
|
+
"vitest": "^5.0.2"
|
|
79
79
|
}
|
|
80
80
|
}
|
|
@@ -4,13 +4,14 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { createReadStream } from "node:fs";
|
|
7
|
-
import { access, mkdir, open, readFile } from "node:fs/promises";
|
|
7
|
+
import { access, lstat, mkdir, open, readFile } from "node:fs/promises";
|
|
8
8
|
import { createInterface } from "node:readline";
|
|
9
9
|
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
|
|
10
10
|
import { loadRehearsalConfig } from "./configuration.mjs";
|
|
11
11
|
import { buildMigrationLedgerInventory } from "./migration_history.mjs";
|
|
12
12
|
|
|
13
13
|
const identifierPattern = /^[a-z][a-z0-9_]{0,62}$/u;
|
|
14
|
+
const storageBucketPattern = /^[a-z0-9][a-z0-9.-]{0,99}$/u;
|
|
14
15
|
|
|
15
16
|
const resolveProjectInput = (projectRoot, value, label) => {
|
|
16
17
|
if (typeof value !== "string" || value.trim() === "") {
|
|
@@ -37,6 +38,19 @@ const pathExists = (path) =>
|
|
|
37
38
|
throw error;
|
|
38
39
|
});
|
|
39
40
|
|
|
41
|
+
const assertRegularInput = async (path, label) => {
|
|
42
|
+
let stats;
|
|
43
|
+
try {
|
|
44
|
+
stats = await lstat(path);
|
|
45
|
+
} catch (error) {
|
|
46
|
+
if (error?.code === "ENOENT") throw new Error(`${label} does not exist.`);
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
if (stats.isSymbolicLink() || !stats.isFile()) {
|
|
50
|
+
throw new Error(`${label} must be a regular project-local file.`);
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
40
54
|
const inspectSyntheticRecords = async (path) => {
|
|
41
55
|
const tables = new Map();
|
|
42
56
|
let rowCount = 0;
|
|
@@ -97,6 +111,120 @@ const inspectSyntheticRecords = async (path) => {
|
|
|
97
111
|
};
|
|
98
112
|
};
|
|
99
113
|
|
|
114
|
+
const inspectAssetManifest = async (path, projectRoot) => {
|
|
115
|
+
let manifest;
|
|
116
|
+
try {
|
|
117
|
+
manifest = JSON.parse(await readFile(path, "utf8"));
|
|
118
|
+
} catch (error) {
|
|
119
|
+
throw new Error("Storage asset manifest contains invalid JSON.", {
|
|
120
|
+
cause: error,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
if (!Array.isArray(manifest)) {
|
|
124
|
+
throw new Error("Storage asset manifest must be a JSON array.");
|
|
125
|
+
}
|
|
126
|
+
const destinations = new Set();
|
|
127
|
+
for (const [index, asset] of manifest.entries()) {
|
|
128
|
+
if (
|
|
129
|
+
!asset ||
|
|
130
|
+
typeof asset !== "object" ||
|
|
131
|
+
Array.isArray(asset) ||
|
|
132
|
+
!storageBucketPattern.test(asset.bucket ?? "") ||
|
|
133
|
+
typeof asset.objectPath !== "string" ||
|
|
134
|
+
!asset.objectPath ||
|
|
135
|
+
asset.objectPath
|
|
136
|
+
.split("/")
|
|
137
|
+
.some((segment) => !segment || segment === "." || segment === "..") ||
|
|
138
|
+
typeof asset.file !== "string" ||
|
|
139
|
+
!asset.file.trim() ||
|
|
140
|
+
(asset.contentType !== undefined && typeof asset.contentType !== "string")
|
|
141
|
+
) {
|
|
142
|
+
throw new Error(
|
|
143
|
+
`Storage asset manifest entry ${index + 1} has an invalid shape.`,
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
const destination = `${asset.bucket}/${asset.objectPath}`;
|
|
147
|
+
if (destinations.has(destination)) {
|
|
148
|
+
throw new Error(`Duplicate Storage asset destination: ${destination}.`);
|
|
149
|
+
}
|
|
150
|
+
destinations.add(destination);
|
|
151
|
+
const assetFile = resolveProjectInput(
|
|
152
|
+
projectRoot,
|
|
153
|
+
asset.file,
|
|
154
|
+
`Storage asset file in entry ${index + 1}`,
|
|
155
|
+
);
|
|
156
|
+
await assertRegularInput(
|
|
157
|
+
assetFile,
|
|
158
|
+
`Storage asset file in entry ${index + 1}`,
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
return { assetCount: manifest.length };
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
export const inspectBaselineInputFiles = async ({
|
|
165
|
+
projectRoot = process.cwd(),
|
|
166
|
+
configPath,
|
|
167
|
+
recordsPath,
|
|
168
|
+
ledgerPath,
|
|
169
|
+
assetsPath,
|
|
170
|
+
}) => {
|
|
171
|
+
const loaded = await loadRehearsalConfig({ projectRoot, configPath });
|
|
172
|
+
const records = resolveProjectInput(
|
|
173
|
+
loaded.projectRoot,
|
|
174
|
+
recordsPath,
|
|
175
|
+
"Synthetic records path",
|
|
176
|
+
);
|
|
177
|
+
const ledger = resolveProjectInput(
|
|
178
|
+
loaded.projectRoot,
|
|
179
|
+
ledgerPath,
|
|
180
|
+
"Migration ledger path",
|
|
181
|
+
);
|
|
182
|
+
const assets = assetsPath
|
|
183
|
+
? resolveProjectInput(
|
|
184
|
+
loaded.projectRoot,
|
|
185
|
+
assetsPath,
|
|
186
|
+
"Storage asset manifest path",
|
|
187
|
+
)
|
|
188
|
+
: undefined;
|
|
189
|
+
await Promise.all([
|
|
190
|
+
assertRegularInput(records, "Synthetic records path"),
|
|
191
|
+
assertRegularInput(ledger, "Migration ledger path"),
|
|
192
|
+
...(assets
|
|
193
|
+
? [assertRegularInput(assets, "Storage asset manifest path")]
|
|
194
|
+
: []),
|
|
195
|
+
]);
|
|
196
|
+
let ledgerRows;
|
|
197
|
+
try {
|
|
198
|
+
ledgerRows = JSON.parse(await readFile(ledger, "utf8"));
|
|
199
|
+
} catch (error) {
|
|
200
|
+
throw new Error("Migration ledger contains invalid JSON.", {
|
|
201
|
+
cause: error,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
const [inspection, assetInspection] = await Promise.all([
|
|
205
|
+
inspectSyntheticRecords(records),
|
|
206
|
+
assets
|
|
207
|
+
? inspectAssetManifest(assets, loaded.projectRoot)
|
|
208
|
+
: Promise.resolve({ assetCount: 0 }),
|
|
209
|
+
]);
|
|
210
|
+
const migrationHistory = buildMigrationLedgerInventory(
|
|
211
|
+
ledgerRows,
|
|
212
|
+
"Synthetic migration ledger",
|
|
213
|
+
);
|
|
214
|
+
return {
|
|
215
|
+
projectRoot: loaded.projectRoot,
|
|
216
|
+
recordsPath: relative(loaded.projectRoot, records),
|
|
217
|
+
ledgerPath: relative(loaded.projectRoot, ledger),
|
|
218
|
+
assetsPath: assets ? relative(loaded.projectRoot, assets) : undefined,
|
|
219
|
+
migrationHistory,
|
|
220
|
+
migrationCutoff: migrationHistory.at(-1).version,
|
|
221
|
+
migrationCount: migrationHistory.length,
|
|
222
|
+
rowCount: inspection.rowCount,
|
|
223
|
+
tables: inspection.tables,
|
|
224
|
+
assetCount: assetInspection.assetCount,
|
|
225
|
+
};
|
|
226
|
+
};
|
|
227
|
+
|
|
100
228
|
const renderPolicyDraft = ({ migrationCutoff, tables }) =>
|
|
101
229
|
`${JSON.stringify(
|
|
102
230
|
{
|
|
@@ -127,43 +255,30 @@ export const planBaselinePreparation = async ({
|
|
|
127
255
|
ledgerPath,
|
|
128
256
|
}) => {
|
|
129
257
|
const loaded = await loadRehearsalConfig({ projectRoot, configPath });
|
|
130
|
-
const
|
|
131
|
-
loaded.projectRoot,
|
|
258
|
+
const inspection = await inspectBaselineInputFiles({
|
|
259
|
+
projectRoot: loaded.projectRoot,
|
|
260
|
+
configPath,
|
|
132
261
|
recordsPath,
|
|
133
|
-
"Synthetic records path",
|
|
134
|
-
);
|
|
135
|
-
const ledger = resolveProjectInput(
|
|
136
|
-
loaded.projectRoot,
|
|
137
262
|
ledgerPath,
|
|
138
|
-
|
|
139
|
-
);
|
|
140
|
-
const [inspection, ledgerRows] = await Promise.all([
|
|
141
|
-
inspectSyntheticRecords(records),
|
|
142
|
-
readFile(ledger, "utf8").then(JSON.parse),
|
|
143
|
-
]);
|
|
144
|
-
const migrationHistory = buildMigrationLedgerInventory(
|
|
145
|
-
ledgerRows,
|
|
146
|
-
"Synthetic migration ledger",
|
|
147
|
-
);
|
|
263
|
+
});
|
|
148
264
|
const destination = loaded.paths.sanitizationPolicy;
|
|
149
265
|
if (await pathExists(destination)) {
|
|
150
266
|
throw new Error(
|
|
151
267
|
`Rehearsal baseline preparation will not overwrite ${relative(loaded.projectRoot, destination)}.`,
|
|
152
268
|
);
|
|
153
269
|
}
|
|
154
|
-
const migrationCutoff = migrationHistory.at(-1).version;
|
|
155
270
|
return {
|
|
156
271
|
projectRoot: loaded.projectRoot,
|
|
157
272
|
destination,
|
|
158
273
|
destinationRelative: relative(loaded.projectRoot, destination),
|
|
159
|
-
recordsPath:
|
|
160
|
-
ledgerPath:
|
|
161
|
-
migrationCutoff,
|
|
162
|
-
migrationCount:
|
|
274
|
+
recordsPath: inspection.recordsPath,
|
|
275
|
+
ledgerPath: inspection.ledgerPath,
|
|
276
|
+
migrationCutoff: inspection.migrationCutoff,
|
|
277
|
+
migrationCount: inspection.migrationCount,
|
|
163
278
|
rowCount: inspection.rowCount,
|
|
164
279
|
tables: inspection.tables,
|
|
165
280
|
content: renderPolicyDraft({
|
|
166
|
-
migrationCutoff,
|
|
281
|
+
migrationCutoff: inspection.migrationCutoff,
|
|
167
282
|
tables: inspection.tables,
|
|
168
283
|
}),
|
|
169
284
|
};
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
import { createHash } from "node:crypto";
|
|
8
8
|
|
|
9
9
|
export const REHEARSAL_RESULT_VERSION = 1;
|
|
10
|
-
export const REHEARSAL_VERSION = "0.1.0-beta.
|
|
10
|
+
export const REHEARSAL_VERSION = "0.1.0-beta.5";
|
|
11
11
|
|
|
12
12
|
export const REHEARSAL_EXIT_CODES = Object.freeze({
|
|
13
13
|
success: 0,
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Discover likely project-owned baseline inputs without exposing their values.
|
|
3
|
+
* Discovery is bounded, ignores generated/private directories, and only sniffs
|
|
4
|
+
* small JSON or NDJSON files before returning project-relative paths.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { open, readFile, readdir, stat } from "node:fs/promises";
|
|
8
|
+
import { extname, join, relative, sep } from "node:path";
|
|
9
|
+
|
|
10
|
+
const ignoredDirectories = new Set([
|
|
11
|
+
".git",
|
|
12
|
+
".next",
|
|
13
|
+
".rehearsal",
|
|
14
|
+
".turbo",
|
|
15
|
+
"build",
|
|
16
|
+
"coverage",
|
|
17
|
+
"dist",
|
|
18
|
+
"node_modules",
|
|
19
|
+
]);
|
|
20
|
+
const maximumDepth = 4;
|
|
21
|
+
const maximumFiles = 2_000;
|
|
22
|
+
const maximumJsonBytes = 2 * 1024 * 1024;
|
|
23
|
+
const maximumNdjsonProbeBytes = 64 * 1024;
|
|
24
|
+
|
|
25
|
+
const projectRelativePath = (projectRoot, path) =>
|
|
26
|
+
relative(projectRoot, path).split(sep).join("/");
|
|
27
|
+
|
|
28
|
+
const preference = {
|
|
29
|
+
records: [
|
|
30
|
+
"rehearsal/sanitized-data.ndjson",
|
|
31
|
+
"rehearsal/synthetic-data.ndjson",
|
|
32
|
+
],
|
|
33
|
+
ledgers: ["rehearsal/migration-ledger.json"],
|
|
34
|
+
assetManifests: ["rehearsal/assets.json"],
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const sortCandidates = (kind, candidates) => {
|
|
38
|
+
const preferred = new Map(
|
|
39
|
+
preference[kind].map((path, index) => [path, index]),
|
|
40
|
+
);
|
|
41
|
+
return candidates.sort((left, right) => {
|
|
42
|
+
const leftRank = preferred.get(left) ?? Number.POSITIVE_INFINITY;
|
|
43
|
+
const rightRank = preferred.get(right) ?? Number.POSITIVE_INFINITY;
|
|
44
|
+
return leftRank - rightRank || left.localeCompare(right);
|
|
45
|
+
});
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
const readFirstRecord = async (path) => {
|
|
49
|
+
const handle = await open(path, "r");
|
|
50
|
+
try {
|
|
51
|
+
const buffer = Buffer.alloc(maximumNdjsonProbeBytes);
|
|
52
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
|
|
53
|
+
const probe = buffer.subarray(0, bytesRead).toString("utf8");
|
|
54
|
+
const lines = probe.split(/\r?\n/u);
|
|
55
|
+
for (const line of lines) {
|
|
56
|
+
if (!line.trim()) continue;
|
|
57
|
+
return JSON.parse(line);
|
|
58
|
+
}
|
|
59
|
+
} finally {
|
|
60
|
+
await handle.close();
|
|
61
|
+
}
|
|
62
|
+
return undefined;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const isRecord = (value) =>
|
|
66
|
+
Boolean(
|
|
67
|
+
value &&
|
|
68
|
+
typeof value === "object" &&
|
|
69
|
+
!Array.isArray(value) &&
|
|
70
|
+
typeof value.table === "string" &&
|
|
71
|
+
value.row &&
|
|
72
|
+
typeof value.row === "object" &&
|
|
73
|
+
!Array.isArray(value.row),
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
const isMigrationLedger = (value) =>
|
|
77
|
+
Array.isArray(value) &&
|
|
78
|
+
value.length > 0 &&
|
|
79
|
+
value.every(
|
|
80
|
+
(entry) =>
|
|
81
|
+
entry &&
|
|
82
|
+
typeof entry === "object" &&
|
|
83
|
+
typeof entry.version === "string" &&
|
|
84
|
+
typeof entry.name === "string" &&
|
|
85
|
+
Array.isArray(entry.statements),
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
const isAssetManifest = (value) =>
|
|
89
|
+
Array.isArray(value) &&
|
|
90
|
+
value.length > 0 &&
|
|
91
|
+
value.every(
|
|
92
|
+
(entry) =>
|
|
93
|
+
entry &&
|
|
94
|
+
typeof entry === "object" &&
|
|
95
|
+
typeof entry.bucket === "string" &&
|
|
96
|
+
typeof entry.objectPath === "string" &&
|
|
97
|
+
typeof entry.file === "string",
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
const collectCandidateFiles = async (projectRoot) => {
|
|
101
|
+
const files = [];
|
|
102
|
+
const visit = async (directory, depth) => {
|
|
103
|
+
if (depth > maximumDepth || files.length >= maximumFiles) return;
|
|
104
|
+
const entries = await readdir(directory, { withFileTypes: true }).catch(
|
|
105
|
+
() => [],
|
|
106
|
+
);
|
|
107
|
+
entries.sort((left, right) => left.name.localeCompare(right.name));
|
|
108
|
+
for (const entry of entries) {
|
|
109
|
+
if (files.length >= maximumFiles) return;
|
|
110
|
+
if (entry.isSymbolicLink()) continue;
|
|
111
|
+
const path = join(directory, entry.name);
|
|
112
|
+
if (entry.isDirectory()) {
|
|
113
|
+
if (!ignoredDirectories.has(entry.name)) await visit(path, depth + 1);
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
if (!entry.isFile() || ![".json", ".ndjson"].includes(extname(path))) {
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
files.push(path);
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
await visit(projectRoot, 0);
|
|
123
|
+
return files;
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
export const discoverBaselineInputFiles = async ({
|
|
127
|
+
projectRoot = process.cwd(),
|
|
128
|
+
} = {}) => {
|
|
129
|
+
const result = { records: [], ledgers: [], assetManifests: [] };
|
|
130
|
+
for (const path of await collectCandidateFiles(projectRoot)) {
|
|
131
|
+
const pathStats = await stat(path).catch(() => undefined);
|
|
132
|
+
if (!pathStats?.isFile()) continue;
|
|
133
|
+
const relativePath = projectRelativePath(projectRoot, path);
|
|
134
|
+
try {
|
|
135
|
+
if (extname(path) === ".ndjson") {
|
|
136
|
+
if (isRecord(await readFirstRecord(path))) {
|
|
137
|
+
result.records.push(relativePath);
|
|
138
|
+
}
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (pathStats.size > maximumJsonBytes) continue;
|
|
142
|
+
const value = JSON.parse(await readFile(path, "utf8"));
|
|
143
|
+
if (isMigrationLedger(value)) result.ledgers.push(relativePath);
|
|
144
|
+
else if (isAssetManifest(value)) result.assetManifests.push(relativePath);
|
|
145
|
+
} catch {
|
|
146
|
+
// Invalid or unreadable files are not candidates; explicit validation will
|
|
147
|
+
// provide actionable errors if the user enters one manually.
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return {
|
|
151
|
+
records: sortCandidates("records", result.records),
|
|
152
|
+
ledgers: sortCandidates("ledgers", result.ledgers),
|
|
153
|
+
assetManifests: sortCandidates("assetManifests", result.assetManifests),
|
|
154
|
+
};
|
|
155
|
+
};
|
|
@@ -7,6 +7,31 @@
|
|
|
7
7
|
import { readFile, writeFile } from "node:fs/promises";
|
|
8
8
|
import { validateRuntimeSanitizationPolicy } from "./sanitization_policy.mjs";
|
|
9
9
|
|
|
10
|
+
export const SAFE_COLUMN_PRESET = Object.freeze({
|
|
11
|
+
action: "REPLACE WITH SYNTHETIC",
|
|
12
|
+
generated: "NEVER",
|
|
13
|
+
identity: "NO",
|
|
14
|
+
foreignKey: null,
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
export const createSafeTablePreset = (columns) =>
|
|
18
|
+
Object.fromEntries(
|
|
19
|
+
columns.map((column) => [
|
|
20
|
+
typeof column === "string" ? column : column.name,
|
|
21
|
+
{ ...SAFE_COLUMN_PRESET },
|
|
22
|
+
]),
|
|
23
|
+
);
|
|
24
|
+
|
|
25
|
+
export const suggestPolicyExceptionColumns = (columns) =>
|
|
26
|
+
columns
|
|
27
|
+
.map((column) => (typeof column === "string" ? column : column.name))
|
|
28
|
+
.filter(
|
|
29
|
+
(column) =>
|
|
30
|
+
column === "id" ||
|
|
31
|
+
column.endsWith("_id") ||
|
|
32
|
+
["created_at", "updated_at", "deleted_at"].includes(column),
|
|
33
|
+
);
|
|
34
|
+
|
|
10
35
|
export const readReviewablePolicyDraft = async (path) => {
|
|
11
36
|
const source = await readFile(path, "utf8");
|
|
12
37
|
const draft = JSON.parse(source);
|
|
@@ -18,18 +43,33 @@ export const readReviewablePolicyDraft = async (path) => {
|
|
|
18
43
|
return { source, draft };
|
|
19
44
|
};
|
|
20
45
|
|
|
21
|
-
export const completePolicyDraft = async ({
|
|
46
|
+
export const completePolicyDraft = async ({
|
|
47
|
+
draft,
|
|
48
|
+
reviewColumn,
|
|
49
|
+
reviewTable,
|
|
50
|
+
}) => {
|
|
22
51
|
if (draft?.draft !== true || !Array.isArray(draft.tables)) {
|
|
23
52
|
throw new Error("A generated REVIEW REQUIRED policy draft is required.");
|
|
24
53
|
}
|
|
25
54
|
const tables = [];
|
|
26
55
|
for (const table of draft.tables) {
|
|
56
|
+
const columnNames = (table.columns ?? []).map((column) => column.name);
|
|
57
|
+
const tableDecisions = reviewTable
|
|
58
|
+
? await reviewTable({ table: table.name, columns: columnNames })
|
|
59
|
+
: {};
|
|
27
60
|
const columns = [];
|
|
28
61
|
for (const column of table.columns ?? []) {
|
|
29
|
-
const reviewed =
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
62
|
+
const reviewed =
|
|
63
|
+
tableDecisions?.[column.name] ??
|
|
64
|
+
(await reviewColumn?.({
|
|
65
|
+
table: table.name,
|
|
66
|
+
column: column.name,
|
|
67
|
+
}));
|
|
68
|
+
if (!reviewed) {
|
|
69
|
+
throw new Error(
|
|
70
|
+
`Policy review did not classify ${table.name}.${column.name}.`,
|
|
71
|
+
);
|
|
72
|
+
}
|
|
33
73
|
columns.push({ name: column.name, ...reviewed });
|
|
34
74
|
}
|
|
35
75
|
tables.push({ ...table, columns });
|