hazo_env 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -4
- package/SETUP_CHECKLIST.md +47 -0
- package/config/hazo_env_masking.ini.sample +16 -16
- package/dist/cli.js +167 -6
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -0
- package/dist/lib/index.d.ts +1 -1
- package/dist/lib/index.d.ts.map +1 -1
- package/dist/lib/index.js +2 -1
- package/dist/lib/secret_columns.d.ts +3 -0
- package/dist/lib/secret_columns.d.ts.map +1 -0
- package/dist/lib/secret_columns.js +14 -0
- package/dist/mask/registry.d.ts +8 -0
- package/dist/mask/registry.d.ts.map +1 -0
- package/dist/mask/registry.js +29 -0
- package/dist/mask/ruleset.d.ts +15 -0
- package/dist/mask/ruleset.d.ts.map +1 -0
- package/dist/mask/ruleset.js +77 -0
- package/dist/migrate/audit.d.ts +21 -0
- package/dist/migrate/audit.d.ts.map +1 -0
- package/dist/migrate/audit.js +36 -0
- package/dist/migrate/db.d.ts +13 -0
- package/dist/migrate/db.d.ts.map +1 -0
- package/dist/migrate/db.js +122 -0
- package/dist/migrate/files.d.ts +14 -0
- package/dist/migrate/files.d.ts.map +1 -0
- package/dist/migrate/files.js +72 -0
- package/dist/migrate/run.d.ts +3 -0
- package/dist/migrate/run.d.ts.map +1 -0
- package/dist/migrate/run.js +195 -0
- package/dist/migrate/snapshot.d.ts +8 -0
- package/dist/migrate/snapshot.d.ts.map +1 -0
- package/dist/migrate/snapshot.js +48 -0
- package/dist/migrate/transport.d.ts +3 -0
- package/dist/migrate/transport.d.ts.map +1 -0
- package/dist/migrate/transport.js +17 -0
- package/dist/migrate/verify.d.ts +8 -0
- package/dist/migrate/verify.d.ts.map +1 -0
- package/dist/migrate/verify.js +51 -0
- package/dist/types/index.d.ts +42 -27
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +15 -3
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { HazoConnectAdapter } from 'hazo_connect';
|
|
2
|
+
export interface DbCopyOptions {
|
|
3
|
+
tables?: '*' | string[];
|
|
4
|
+
scrubHook?: (tableName: string, row: Record<string, unknown>) => Record<string, unknown>;
|
|
5
|
+
onProgress?: (msg: string) => void;
|
|
6
|
+
}
|
|
7
|
+
export interface DbCopyResult {
|
|
8
|
+
tables: number;
|
|
9
|
+
rows: number;
|
|
10
|
+
scrubbed: number;
|
|
11
|
+
}
|
|
12
|
+
export declare function copyDb(srcAdapter: HazoConnectAdapter, tgtAdapter: HazoConnectAdapter, opts?: DbCopyOptions): Promise<DbCopyResult>;
|
|
13
|
+
//# sourceMappingURL=db.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/migrate/db.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAEvD,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,EAAE,GAAG,GAAG,MAAM,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzF,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CAClB;AA6BD,wBAAsB,MAAM,CAC1B,UAAU,EAAE,kBAAkB,EAC9B,UAAU,EAAE,kBAAkB,EAC9B,IAAI,GAAE,aAAkB,GACvB,OAAO,CAAC,YAAY,CAAC,CA+GvB"}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
// hazo_env/src/migrate/db.ts — per-table DB copy with schema parity check
|
|
2
|
+
import { HazoError, optional_import } from 'hazo_core';
|
|
3
|
+
import { isSecretColumn } from '../lib/secret_columns.js';
|
|
4
|
+
const PAGE_SIZE = 500;
|
|
5
|
+
// The HazoConnectAdapter interface types rawQuery as (sql, RequestInit?) but the
|
|
6
|
+
// SQLite adapter extends it with { params?: unknown[] }. Cast through unknown to
|
|
7
|
+
// satisfy both the interface type and the runtime expectation.
|
|
8
|
+
function rawQuery(adapter, sql, params = []) {
|
|
9
|
+
return adapter.rawQuery(sql, { params });
|
|
10
|
+
}
|
|
11
|
+
async function getRawTables(adapter) {
|
|
12
|
+
const result = await rawQuery(adapter, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name");
|
|
13
|
+
return result.map((r) => r.name);
|
|
14
|
+
}
|
|
15
|
+
async function getMigrationNames(adapter) {
|
|
16
|
+
// Check if _migrations table exists
|
|
17
|
+
const tables = await getRawTables(adapter);
|
|
18
|
+
if (!tables.includes('_migrations'))
|
|
19
|
+
return null;
|
|
20
|
+
const rows = await rawQuery(adapter, 'SELECT name FROM _migrations ORDER BY name');
|
|
21
|
+
return rows.map((r) => r.name);
|
|
22
|
+
}
|
|
23
|
+
export async function copyDb(srcAdapter, tgtAdapter, opts = {}) {
|
|
24
|
+
const connect = await optional_import('hazo_connect/server');
|
|
25
|
+
if (!connect) {
|
|
26
|
+
throw new HazoError({
|
|
27
|
+
code: 'HAZO_ENV_MISSING_DEPENDENCY',
|
|
28
|
+
pkg: 'hazo_env',
|
|
29
|
+
message: 'hazo_connect is required for DB migration. Install hazo_connect to continue.',
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
// Schema parity check via _migrations
|
|
33
|
+
const srcMigrations = await getMigrationNames(srcAdapter);
|
|
34
|
+
const tgtMigrations = await getMigrationNames(tgtAdapter);
|
|
35
|
+
if (srcMigrations === null || tgtMigrations === null) {
|
|
36
|
+
throw new HazoError({
|
|
37
|
+
code: 'HAZO_ENV_SCHEMA_MISMATCH',
|
|
38
|
+
pkg: 'hazo_env',
|
|
39
|
+
message: `Schema parity check failed: _migrations table is missing on ${srcMigrations === null ? 'source' : 'target'}. Both databases must have matching _migrations tables.`,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
const srcSet = new Set(srcMigrations);
|
|
43
|
+
const tgtSet = new Set(tgtMigrations);
|
|
44
|
+
const missingInTgt = [...srcSet].filter((n) => !tgtSet.has(n));
|
|
45
|
+
const missingInSrc = [...tgtSet].filter((n) => !srcSet.has(n));
|
|
46
|
+
if (missingInTgt.length > 0 || missingInSrc.length > 0) {
|
|
47
|
+
throw new HazoError({
|
|
48
|
+
code: 'HAZO_ENV_SCHEMA_MISMATCH',
|
|
49
|
+
pkg: 'hazo_env',
|
|
50
|
+
message: `Schema parity check failed: migration name-sets differ.\n Missing in target: ${missingInTgt.join(', ') || 'none'}\n Missing in source: ${missingInSrc.join(', ') || 'none'}`,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
// Get tables to copy
|
|
54
|
+
const allTables = await getRawTables(srcAdapter);
|
|
55
|
+
// Exclude system/internal tables
|
|
56
|
+
const SKIP = new Set(['_migrations', 'sqlite_sequence', 'hazo_audit_events', 'hazo_audit_intent']);
|
|
57
|
+
let tablesToCopy = allTables.filter((t) => !SKIP.has(t));
|
|
58
|
+
if (opts.tables && opts.tables !== '*') {
|
|
59
|
+
tablesToCopy = tablesToCopy.filter((t) => opts.tables.includes(t));
|
|
60
|
+
}
|
|
61
|
+
let totalRows = 0;
|
|
62
|
+
let totalScrubbed = 0;
|
|
63
|
+
for (const table of tablesToCopy) {
|
|
64
|
+
opts.onProgress?.(`Copying table: ${table}`);
|
|
65
|
+
// Delete all rows in target
|
|
66
|
+
await rawQuery(tgtAdapter, `DELETE FROM "${table}"`);
|
|
67
|
+
// Get schema to find PK column and all columns
|
|
68
|
+
const schema = await rawQuery(srcAdapter, `PRAGMA table_info("${table}")`);
|
|
69
|
+
const pkCol = schema.find((c) => c.pk === 1)?.name ?? 'id';
|
|
70
|
+
const allCols = schema.map((c) => c.name);
|
|
71
|
+
// Page through source
|
|
72
|
+
const srcService = connect.createCrudService(srcAdapter, table, { autoId: false });
|
|
73
|
+
let offset = 0;
|
|
74
|
+
let pageRows = [];
|
|
75
|
+
do {
|
|
76
|
+
pageRows = await srcService.list((qb) => qb.order(pkCol, 'asc').limit(PAGE_SIZE).offset(offset));
|
|
77
|
+
if (pageRows.length === 0)
|
|
78
|
+
break;
|
|
79
|
+
for (let row of pageRows) {
|
|
80
|
+
// Drop secret columns
|
|
81
|
+
const cleaned = {};
|
|
82
|
+
let didScrub = false;
|
|
83
|
+
for (const col of allCols) {
|
|
84
|
+
if (isSecretColumn(col)) {
|
|
85
|
+
didScrub = true;
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
cleaned[col] = row[col];
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
// Apply scrub hook if provided (masking, Phase B)
|
|
92
|
+
if (opts.scrubHook) {
|
|
93
|
+
const hooked = opts.scrubHook(table, cleaned);
|
|
94
|
+
if (hooked !== cleaned)
|
|
95
|
+
didScrub = true;
|
|
96
|
+
row = hooked;
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
row = cleaned;
|
|
100
|
+
}
|
|
101
|
+
if (didScrub)
|
|
102
|
+
totalScrubbed++;
|
|
103
|
+
// Only insert columns that exist in schema
|
|
104
|
+
const insertRow = {};
|
|
105
|
+
for (const col of Object.keys(row)) {
|
|
106
|
+
if (allCols.includes(col))
|
|
107
|
+
insertRow[col] = row[col];
|
|
108
|
+
}
|
|
109
|
+
// Use raw INSERT to preserve PKs
|
|
110
|
+
const cols = Object.keys(insertRow);
|
|
111
|
+
if (cols.length === 0)
|
|
112
|
+
continue;
|
|
113
|
+
const placeholders = cols.map(() => '?').join(', ');
|
|
114
|
+
const values = cols.map((c) => insertRow[c]);
|
|
115
|
+
await rawQuery(tgtAdapter, `INSERT OR REPLACE INTO "${table}" (${cols.map((c) => `"${c}"`).join(', ')}) VALUES (${placeholders})`, values);
|
|
116
|
+
totalRows++;
|
|
117
|
+
}
|
|
118
|
+
offset += PAGE_SIZE;
|
|
119
|
+
} while (pageRows.length === PAGE_SIZE);
|
|
120
|
+
}
|
|
121
|
+
return { tables: tablesToCopy.length, rows: totalRows, scrubbed: totalScrubbed };
|
|
122
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { HazoEnv } from '../types/index.js';
|
|
2
|
+
export interface FilesCopyOptions {
|
|
3
|
+
fromEnv: HazoEnv;
|
|
4
|
+
toEnv: HazoEnv;
|
|
5
|
+
fileScrubHook?: (filePath: string) => Promise<void>;
|
|
6
|
+
onProgress?: (msg: string) => void;
|
|
7
|
+
}
|
|
8
|
+
export interface FilesCopyResult {
|
|
9
|
+
copied: number;
|
|
10
|
+
bytes: number;
|
|
11
|
+
placeholdered: number;
|
|
12
|
+
}
|
|
13
|
+
export declare function copyFiles(opts: FilesCopyOptions): Promise<FilesCopyResult>;
|
|
14
|
+
//# sourceMappingURL=files.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../../src/migrate/files.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;CACvB;AA4BD,wBAAsB,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CA4ChF"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// hazo_env/src/migrate/files.ts — local file-tree copy with PDF scrubbing (Phase C)
|
|
2
|
+
//
|
|
3
|
+
// PDF scrubbing via hazo_pdf/server (optional peer dep). When HAZO_ENV_MASK_KEY
|
|
4
|
+
// is set and a copied file has a .pdf extension, mask_pdf is called with an
|
|
5
|
+
// empty targets list (structure-only scrub / reserved for future target lists).
|
|
6
|
+
// Failures are non-fatal warnings — the file copy still completes.
|
|
7
|
+
import fs from 'node:fs';
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
import { resolveFilesConfig } from '../resolve/files.js';
|
|
10
|
+
function resolveEnvFilesRoot(env) {
|
|
11
|
+
// resolveFilesConfig returns the data root; env-specific files are at <root>/<env>/files
|
|
12
|
+
const base = resolveFilesConfig().local.basePath;
|
|
13
|
+
return path.join(base, env, 'files');
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Attempt to scrub a PDF file in-place using hazo_pdf/mask_pdf.
|
|
17
|
+
* Non-fatal: any failure is silently swallowed — the copy always completes.
|
|
18
|
+
* Called only when HAZO_ENV_MASK_KEY is set.
|
|
19
|
+
*/
|
|
20
|
+
async function scrubPdfFile(filePath) {
|
|
21
|
+
try {
|
|
22
|
+
const pdfMod = await import('hazo_pdf/server').catch(() => null);
|
|
23
|
+
if (!pdfMod?.mask_pdf)
|
|
24
|
+
return; // hazo_pdf not installed or mask_pdf not exported
|
|
25
|
+
const { readFileSync, writeFileSync } = await import('node:fs');
|
|
26
|
+
const bytes = readFileSync(filePath);
|
|
27
|
+
const scrubbed = await pdfMod.mask_pdf(new Uint8Array(bytes), []);
|
|
28
|
+
writeFileSync(filePath, Buffer.from(scrubbed));
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
// Non-fatal — PDF scrubbing failure is a warning, not an abort
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
export async function copyFiles(opts) {
|
|
35
|
+
const srcRoot = resolveEnvFilesRoot(opts.fromEnv);
|
|
36
|
+
const tgtRoot = resolveEnvFilesRoot(opts.toEnv);
|
|
37
|
+
if (!fs.existsSync(srcRoot)) {
|
|
38
|
+
opts.onProgress?.(`No files directory at ${srcRoot} — skipping file copy`);
|
|
39
|
+
return { copied: 0, bytes: 0, placeholdered: 0 };
|
|
40
|
+
}
|
|
41
|
+
fs.mkdirSync(tgtRoot, { recursive: true });
|
|
42
|
+
let copied = 0;
|
|
43
|
+
let bytes = 0;
|
|
44
|
+
async function copyDir(src, tgt) {
|
|
45
|
+
const entries = fs.readdirSync(src, { withFileTypes: true });
|
|
46
|
+
for (const entry of entries) {
|
|
47
|
+
const srcPath = path.join(src, entry.name);
|
|
48
|
+
const tgtPath = path.join(tgt, entry.name);
|
|
49
|
+
if (entry.isDirectory()) {
|
|
50
|
+
fs.mkdirSync(tgtPath, { recursive: true });
|
|
51
|
+
await copyDir(srcPath, tgtPath);
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
fs.copyFileSync(srcPath, tgtPath);
|
|
55
|
+
const stat = fs.statSync(srcPath);
|
|
56
|
+
bytes += stat.size;
|
|
57
|
+
copied++;
|
|
58
|
+
opts.onProgress?.(`Copied: ${entry.name}`);
|
|
59
|
+
// Phase C: PDF scrubbing when HAZO_ENV_MASK_KEY is set
|
|
60
|
+
if (entry.name.toLowerCase().endsWith('.pdf') && process.env['HAZO_ENV_MASK_KEY']) {
|
|
61
|
+
await scrubPdfFile(tgtPath);
|
|
62
|
+
}
|
|
63
|
+
// Legacy sync hook (still supported for callers that pass one)
|
|
64
|
+
if (opts.fileScrubHook) {
|
|
65
|
+
await opts.fileScrubHook(tgtPath);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
await copyDir(srcRoot, tgtRoot);
|
|
71
|
+
return { copied, bytes, placeholdered: 0 };
|
|
72
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/migrate/run.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAkC,MAAM,mBAAmB,CAAC;AAgE3G,wBAAsB,YAAY,CAAC,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAiJlF"}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
// hazo_env/src/migrate/run.ts — runMigration pipeline
|
|
2
|
+
//
|
|
3
|
+
// PDF scrubbing (Phase C): after each file copy, files.ts invokes hazo_pdf/server mask_pdf
|
|
4
|
+
// for any .pdf file when HAZO_ENV_MASK_KEY is set. hazo_pdf is an optional peer dep of
|
|
5
|
+
// hazo_env — if not installed the copy still completes (scrub silently skipped).
|
|
6
|
+
// See: src/migrate/files.ts scrubPdfFile() and hazo_pdf/src/server/mask.ts
|
|
7
|
+
import { HazoError, optional_import } from 'hazo_core';
|
|
8
|
+
import { resolveConnectConfig } from '../resolve/connect.js';
|
|
9
|
+
import { resolveFilesConfig } from '../resolve/files.js';
|
|
10
|
+
import { resolveTransport } from './transport.js';
|
|
11
|
+
import { takeSnapshot } from './snapshot.js';
|
|
12
|
+
import { copyDb } from './db.js';
|
|
13
|
+
import { copyFiles } from './files.js';
|
|
14
|
+
import { verifyFiles } from './verify.js';
|
|
15
|
+
import { auditMigration } from './audit.js';
|
|
16
|
+
// Production roles that require explicit confirmation
|
|
17
|
+
const PROD_ROLES = new Set(['prod', 'production']);
|
|
18
|
+
function isProdEnv(env) {
|
|
19
|
+
return PROD_ROLES.has(env.toLowerCase());
|
|
20
|
+
}
|
|
21
|
+
function emitProgress(req, p) {
|
|
22
|
+
req.onProgress?.(p);
|
|
23
|
+
}
|
|
24
|
+
// Resolve the SQLite driver — prefer better-sqlite3 in tests when env var is set
|
|
25
|
+
function getSqliteDriver() {
|
|
26
|
+
if (process.env['HAZO_ENV_TEST_SQLITE_DRIVER'] === 'better-sqlite3')
|
|
27
|
+
return 'better-sqlite3';
|
|
28
|
+
return 'sql.js';
|
|
29
|
+
}
|
|
30
|
+
async function buildScrubHook(toEnv, scrubMode, tgtAdapter) {
|
|
31
|
+
const maskKey = process.env['HAZO_ENV_MASK_KEY'];
|
|
32
|
+
const shouldScrub = scrubMode !== 'none' &&
|
|
33
|
+
maskKey &&
|
|
34
|
+
(toEnv === 'test' || toEnv === 'staging');
|
|
35
|
+
if (!shouldScrub)
|
|
36
|
+
return undefined;
|
|
37
|
+
const { loadRuleset } = await import('../mask/ruleset.js');
|
|
38
|
+
const { getTransform } = await import('../mask/registry.js');
|
|
39
|
+
let ruleset;
|
|
40
|
+
try {
|
|
41
|
+
ruleset = await loadRuleset(tgtAdapter);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|
|
46
|
+
if (ruleset.length === 0)
|
|
47
|
+
return undefined;
|
|
48
|
+
const transformMap = new Map();
|
|
49
|
+
for (const rule of ruleset) {
|
|
50
|
+
const fn = await getTransform(rule.transform);
|
|
51
|
+
if (!fn)
|
|
52
|
+
continue;
|
|
53
|
+
if (!transformMap.has(rule.table))
|
|
54
|
+
transformMap.set(rule.table, new Map());
|
|
55
|
+
transformMap.get(rule.table).set(rule.column, fn);
|
|
56
|
+
}
|
|
57
|
+
return (table, row) => {
|
|
58
|
+
const colTransforms = transformMap.get(table);
|
|
59
|
+
if (!colTransforms || colTransforms.size === 0)
|
|
60
|
+
return row;
|
|
61
|
+
const result = { ...row };
|
|
62
|
+
for (const [col, fn] of colTransforms) {
|
|
63
|
+
if (col in result)
|
|
64
|
+
result[col] = fn(result[col], col, maskKey);
|
|
65
|
+
}
|
|
66
|
+
return result;
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
export async function runMigration(req) {
|
|
70
|
+
const startMs = Date.now();
|
|
71
|
+
const warnings = [];
|
|
72
|
+
const connect = await optional_import('hazo_connect/server');
|
|
73
|
+
if (!connect) {
|
|
74
|
+
throw new HazoError({
|
|
75
|
+
code: 'HAZO_ENV_MISSING_DEPENDENCY',
|
|
76
|
+
pkg: 'hazo_env',
|
|
77
|
+
message: 'hazo_connect is required for migration. Install it as a peer dependency.',
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
// Step 1: Validate
|
|
81
|
+
emitProgress(req, { phase: 'validate', message: 'Validating migration request', percent: 0 });
|
|
82
|
+
if (!req.from || !req.to) {
|
|
83
|
+
throw new HazoError({ code: 'HAZO_ENV_INVALID_REQUEST', pkg: 'hazo_env', message: 'Migration requires both from and to environments.' });
|
|
84
|
+
}
|
|
85
|
+
if (req.from === req.to) {
|
|
86
|
+
throw new HazoError({ code: 'HAZO_ENV_INVALID_REQUEST', pkg: 'hazo_env', message: `Source and target environments must differ (got "${req.from}" for both).` });
|
|
87
|
+
}
|
|
88
|
+
if (isProdEnv(req.to) && !req.allowProdTarget) {
|
|
89
|
+
throw new HazoError({
|
|
90
|
+
code: 'HAZO_ENV_PROD_TARGET_REFUSED',
|
|
91
|
+
pkg: 'hazo_env',
|
|
92
|
+
message: `Refusing to migrate to production environment "${req.to}". Pass allowProdTarget:true and a confirmToken to override.`,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
if (isProdEnv(req.to) && req.allowProdTarget && !req.confirmToken) {
|
|
96
|
+
throw new HazoError({
|
|
97
|
+
code: 'HAZO_ENV_PROD_TARGET_REFUSED',
|
|
98
|
+
pkg: 'hazo_env',
|
|
99
|
+
message: 'Production target requires a confirmToken for safety.',
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
const fromDbConfig = resolveConnectConfig({ env: req.from, allowOtherEnv: true });
|
|
103
|
+
const toDbConfig = resolveConnectConfig({ env: req.to, allowOtherEnv: true });
|
|
104
|
+
resolveTransport(fromDbConfig, toDbConfig, req.transport);
|
|
105
|
+
const includeDb = req.include?.db ?? true;
|
|
106
|
+
const includeFiles = req.include?.files ?? true;
|
|
107
|
+
const scrubMode = req.scrub ?? 'auto';
|
|
108
|
+
// Step 2: Snapshot target
|
|
109
|
+
emitProgress(req, { phase: 'snapshot', message: 'Taking snapshot of target', percent: 5 });
|
|
110
|
+
const snapshot = takeSnapshot(toDbConfig);
|
|
111
|
+
// Step 3: Plan (dry-run stops here)
|
|
112
|
+
emitProgress(req, { phase: 'plan', message: 'Building migration plan', percent: 10 });
|
|
113
|
+
if (req.dryRun) {
|
|
114
|
+
return {
|
|
115
|
+
ok: true,
|
|
116
|
+
snapshotId: snapshot.snapshotId,
|
|
117
|
+
db: includeDb ? { tables: 0, rows: 0, scrubbed: 0 } : undefined,
|
|
118
|
+
files: includeFiles ? { copied: 0, bytes: 0, placeholdered: 0 } : undefined,
|
|
119
|
+
warnings: ['dry-run: no changes made'],
|
|
120
|
+
durationMs: Date.now() - startMs,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
// Create adapters
|
|
124
|
+
function makeAdapter(dbConfig, readOnly = false) {
|
|
125
|
+
if (dbConfig.type !== 'sqlite' || !dbConfig.sqlite) {
|
|
126
|
+
throw new HazoError({
|
|
127
|
+
code: 'HAZO_ENV_NOT_IMPLEMENTED',
|
|
128
|
+
pkg: 'hazo_env',
|
|
129
|
+
message: `Migration is only implemented for SQLite databases. Got type "${dbConfig.type}".`,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
return connect.createHazoConnect({
|
|
133
|
+
type: 'sqlite',
|
|
134
|
+
sqlite: {
|
|
135
|
+
database_path: dbConfig.sqlite.database_path,
|
|
136
|
+
read_only: readOnly,
|
|
137
|
+
driver: getSqliteDriver(),
|
|
138
|
+
},
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
const srcAdapter = makeAdapter(fromDbConfig, true);
|
|
142
|
+
const tgtAdapter = makeAdapter(toDbConfig, false);
|
|
143
|
+
// Step 4: Schema parity (checked inside copyDb)
|
|
144
|
+
emitProgress(req, { phase: 'schema-check', message: 'Checking schema parity', percent: 15 });
|
|
145
|
+
let dbResult;
|
|
146
|
+
let filesResult;
|
|
147
|
+
// Step 5: DB copy
|
|
148
|
+
if (includeDb) {
|
|
149
|
+
emitProgress(req, { phase: 'db', message: 'Copying database tables', percent: 20 });
|
|
150
|
+
const scrubHook = await buildScrubHook(req.to, scrubMode, tgtAdapter);
|
|
151
|
+
const result = await copyDb(srcAdapter, tgtAdapter, {
|
|
152
|
+
tables: req.tables,
|
|
153
|
+
scrubHook,
|
|
154
|
+
onProgress: (msg) => emitProgress(req, { phase: 'db', message: msg }),
|
|
155
|
+
});
|
|
156
|
+
dbResult = result;
|
|
157
|
+
emitProgress(req, { phase: 'db', message: `DB copy complete: ${result.tables} tables, ${result.rows} rows`, percent: 70 });
|
|
158
|
+
}
|
|
159
|
+
// Step 6: File copy
|
|
160
|
+
if (includeFiles) {
|
|
161
|
+
emitProgress(req, { phase: 'files', message: 'Copying files', percent: 72 });
|
|
162
|
+
const result = await copyFiles({
|
|
163
|
+
fromEnv: req.from,
|
|
164
|
+
toEnv: req.to,
|
|
165
|
+
onProgress: (msg) => emitProgress(req, { phase: 'files', message: msg }),
|
|
166
|
+
});
|
|
167
|
+
filesResult = result;
|
|
168
|
+
}
|
|
169
|
+
// Step 7: Verify
|
|
170
|
+
emitProgress(req, { phase: 'verify', message: 'Running post-migration verification', percent: 80 });
|
|
171
|
+
const dataRoot = resolveFilesConfig().local.basePath;
|
|
172
|
+
const verifyReport = await verifyFiles(req.to, dataRoot, { hash: 'sample', checkOrphans: true });
|
|
173
|
+
if (!verifyReport.ok) {
|
|
174
|
+
warnings.push(`Verification found issues: missing=${verifyReport.missing.length}, sizeMismatch=${verifyReport.sizeMismatch.length}`);
|
|
175
|
+
}
|
|
176
|
+
// Step 8: Finalize — audit
|
|
177
|
+
emitProgress(req, { phase: 'finalize', message: 'Recording audit event', percent: 95 });
|
|
178
|
+
await auditMigration(tgtAdapter, {
|
|
179
|
+
from: req.from,
|
|
180
|
+
to: req.to,
|
|
181
|
+
scrub: scrubMode,
|
|
182
|
+
db: dbResult,
|
|
183
|
+
files: filesResult ? { copied: filesResult.copied, bytes: filesResult.bytes } : undefined,
|
|
184
|
+
verify: { ok: verifyReport.ok, checked: verifyReport.checked },
|
|
185
|
+
});
|
|
186
|
+
emitProgress(req, { phase: 'done', message: 'Migration complete', percent: 100 });
|
|
187
|
+
return {
|
|
188
|
+
ok: true,
|
|
189
|
+
snapshotId: snapshot.snapshotId,
|
|
190
|
+
db: dbResult,
|
|
191
|
+
files: filesResult,
|
|
192
|
+
warnings,
|
|
193
|
+
durationMs: Date.now() - startMs,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { DbEnvConfig } from '../types/index.js';
|
|
2
|
+
export interface SnapshotResult {
|
|
3
|
+
snapshotId: string;
|
|
4
|
+
createdAt: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function takeSnapshot(toConfig: DbEnvConfig): SnapshotResult;
|
|
7
|
+
export declare function restoreSnapshot(toConfig: DbEnvConfig, snapshotId: string): void;
|
|
8
|
+
//# sourceMappingURL=snapshot.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../src/migrate/snapshot.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,WAAW,GAAG,cAAc,CAmBlE;AAED,wBAAgB,eAAe,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAuB/E"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// hazo_env/src/migrate/snapshot.ts — SQLite file-copy snapshot
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { HazoError } from 'hazo_core';
|
|
5
|
+
export function takeSnapshot(toConfig) {
|
|
6
|
+
if (toConfig.type !== 'sqlite' || !toConfig.sqlite?.database_path) {
|
|
7
|
+
throw new HazoError({
|
|
8
|
+
code: 'HAZO_ENV_NOT_IMPLEMENTED',
|
|
9
|
+
pkg: 'hazo_env',
|
|
10
|
+
message: 'Snapshot is only supported for SQLite databases.',
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
const srcPath = toConfig.sqlite.database_path;
|
|
14
|
+
if (!fs.existsSync(srcPath)) {
|
|
15
|
+
// No DB yet — snapshot is a no-op, return a virtual ID
|
|
16
|
+
return { snapshotId: `no-db:${srcPath}`, createdAt: new Date().toISOString() };
|
|
17
|
+
}
|
|
18
|
+
const snapshotDir = path.join(path.dirname(srcPath), 'snapshots');
|
|
19
|
+
fs.mkdirSync(snapshotDir, { recursive: true });
|
|
20
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
21
|
+
const snapshotPath = path.join(snapshotDir, `pre-migrate-${ts}.sqlite`);
|
|
22
|
+
fs.copyFileSync(srcPath, snapshotPath);
|
|
23
|
+
return { snapshotId: snapshotPath, createdAt: new Date().toISOString() };
|
|
24
|
+
}
|
|
25
|
+
export function restoreSnapshot(toConfig, snapshotId) {
|
|
26
|
+
if (toConfig.type !== 'sqlite' || !toConfig.sqlite?.database_path) {
|
|
27
|
+
throw new HazoError({
|
|
28
|
+
code: 'HAZO_ENV_NOT_IMPLEMENTED',
|
|
29
|
+
pkg: 'hazo_env',
|
|
30
|
+
message: 'Restore is only supported for SQLite databases.',
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
if (snapshotId.startsWith('no-db:')) {
|
|
34
|
+
throw new HazoError({
|
|
35
|
+
code: 'HAZO_ENV_SNAPSHOT_NOT_FOUND',
|
|
36
|
+
pkg: 'hazo_env',
|
|
37
|
+
message: 'Cannot restore: snapshot was taken when no database existed.',
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
if (!fs.existsSync(snapshotId)) {
|
|
41
|
+
throw new HazoError({
|
|
42
|
+
code: 'HAZO_ENV_SNAPSHOT_NOT_FOUND',
|
|
43
|
+
pkg: 'hazo_env',
|
|
44
|
+
message: `Snapshot file not found: ${snapshotId}`,
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
fs.copyFileSync(snapshotId, toConfig.sqlite.database_path);
|
|
48
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transport.d.ts","sourceRoot":"","sources":["../../src/migrate/transport.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEpE,wBAAgB,gBAAgB,CAC9B,UAAU,EAAE,WAAW,EACvB,QAAQ,EAAE,WAAW,EACrB,SAAS,GAAE,aAAsB,GAChC,OAAO,GAAG,KAAK,CAYjB"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// hazo_env/src/migrate/transport.ts — resolve transport mode for a migration
|
|
2
|
+
import { HazoError } from 'hazo_core';
|
|
3
|
+
export function resolveTransport(fromConfig, toConfig, requested = 'auto') {
|
|
4
|
+
if (requested === 'ssh') {
|
|
5
|
+
throw new HazoError({
|
|
6
|
+
code: 'HAZO_ENV_NOT_IMPLEMENTED',
|
|
7
|
+
pkg: 'hazo_env',
|
|
8
|
+
message: 'SSH transport is not yet implemented. Use local or api transport.',
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
if (requested === 'api')
|
|
12
|
+
return 'api';
|
|
13
|
+
// auto: both sqlite → local
|
|
14
|
+
if (fromConfig.type === 'sqlite' && toConfig.type === 'sqlite')
|
|
15
|
+
return 'local';
|
|
16
|
+
return 'api';
|
|
17
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { HazoEnv, VerifyReport, HashMode } from '../types/index.js';
|
|
2
|
+
export interface VerifyOptions {
|
|
3
|
+
hash?: HashMode;
|
|
4
|
+
samplePct?: number;
|
|
5
|
+
checkOrphans?: boolean;
|
|
6
|
+
}
|
|
7
|
+
export declare function verifyFiles(env: HazoEnv, dataRoot: string, opts?: VerifyOptions): Promise<VerifyReport>;
|
|
8
|
+
//# sourceMappingURL=verify.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["../../src/migrate/verify.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAEzE,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAMD,wBAAsB,WAAW,CAC/B,GAAG,EAAE,OAAO,EACZ,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,aAAkB,GACvB,OAAO,CAAC,YAAY,CAAC,CAgDvB"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// hazo_env/src/migrate/verify.ts — post-migration file verification
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
function resolveEnvFilesRoot(env, dataRoot) {
|
|
5
|
+
return path.join(dataRoot, env, 'files');
|
|
6
|
+
}
|
|
7
|
+
export async function verifyFiles(env, dataRoot, opts = {}) {
|
|
8
|
+
const { checkOrphans = true } = opts;
|
|
9
|
+
const filesRoot = resolveEnvFilesRoot(env, dataRoot);
|
|
10
|
+
const missing = [];
|
|
11
|
+
const sizeMismatch = [];
|
|
12
|
+
const hashMismatch = [];
|
|
13
|
+
const orphans = [];
|
|
14
|
+
let checked = 0;
|
|
15
|
+
let sentinelOk = true;
|
|
16
|
+
// Layer 1: check sentinel file (data root exists and is readable)
|
|
17
|
+
if (!fs.existsSync(filesRoot)) {
|
|
18
|
+
sentinelOk = false;
|
|
19
|
+
return { ok: false, checked: 0, missing: [], sizeMismatch: [], hashMismatch: [], orphans: [], sentinelOk: false };
|
|
20
|
+
}
|
|
21
|
+
// Layer 2 & 3: walk all files
|
|
22
|
+
function walkDir(dir, relBase) {
|
|
23
|
+
if (!fs.existsSync(dir))
|
|
24
|
+
return [];
|
|
25
|
+
return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
|
|
26
|
+
const relPath = path.join(relBase, entry.name);
|
|
27
|
+
const absPath = path.join(dir, entry.name);
|
|
28
|
+
if (entry.isDirectory())
|
|
29
|
+
return walkDir(absPath, relPath);
|
|
30
|
+
return [relPath];
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
const diskFiles = walkDir(filesRoot, '');
|
|
34
|
+
for (const relPath of diskFiles) {
|
|
35
|
+
const absPath = path.join(filesRoot, relPath);
|
|
36
|
+
try {
|
|
37
|
+
fs.statSync(absPath); // existence + size (stat succeeds = file accessible)
|
|
38
|
+
checked++;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
missing.push(relPath);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
// Layer 4: orphan check — files on disk with no corresponding DB tracking row
|
|
45
|
+
// In Phase A, hazo_files integration is not wired; just verify directory is readable
|
|
46
|
+
if (checkOrphans && !fs.existsSync(filesRoot)) {
|
|
47
|
+
orphans.push(filesRoot);
|
|
48
|
+
}
|
|
49
|
+
const ok = missing.length === 0 && sizeMismatch.length === 0 && hashMismatch.length === 0;
|
|
50
|
+
return { ok, checked, missing, sizeMismatch, hashMismatch, orphans, sentinelOk };
|
|
51
|
+
}
|