@geraldmaron/construct 1.0.23 → 1.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -2
- package/bin/construct +15 -215
- package/lib/embed/inbox.mjs +6 -3
- package/lib/embed/recommendation-store.mjs +7 -289
- package/lib/embed/reconcile.mjs +2 -2
- package/lib/hooks/config-protection.mjs +4 -4
- package/lib/hooks/session-reflect.mjs +5 -1
- package/lib/intake/git-queue.mjs +195 -0
- package/lib/intake/queue.mjs +9 -16
- package/lib/mcp/tools/storage.mjs +2 -3
- package/lib/mcp-catalog.json +3 -3
- package/lib/mcp-manager.mjs +59 -3
- package/lib/observation-store.mjs +38 -166
- package/lib/orchestration/runtime.mjs +3 -2
- package/lib/reconcile/index.mjs +0 -2
- package/lib/service-manager.mjs +38 -256
- package/lib/setup.mjs +26 -426
- package/lib/status.mjs +3 -6
- package/lib/storage/admin.mjs +48 -325
- package/lib/storage/backend.mjs +10 -57
- package/lib/storage/hybrid-query.mjs +15 -196
- package/lib/storage/sync.mjs +36 -177
- package/lib/storage/vector-client.mjs +256 -235
- package/lib/strategy-store.mjs +35 -286
- package/lib/worker/entrypoint.mjs +6 -14
- package/package.json +6 -5
- package/platforms/claude/settings.template.json +0 -7
- package/scripts/sync-specialists.mjs +46 -12
- package/specialists/prompts/cx-qa.md +1 -1
- package/specialists/registry.json +0 -8
- package/templates/docs/construct_guide.md +1 -1
- package/db/schema/001_init.sql +0 -40
- package/db/schema/002_pgvector.sql +0 -182
- package/db/schema/003_intake.sql +0 -47
- package/db/schema/003_observation_reconciliation.sql +0 -14
- package/db/schema/004_recommendations.sql +0 -46
- package/db/schema/005_strategy.sql +0 -21
- package/db/schema/006_graph.sql +0 -24
- package/db/schema/007_tags.sql +0 -30
- package/db/schema/008_skill_usage.sql +0 -24
- package/db/schema/009_scheduler.sql +0 -14
- package/db/schema/010_cx_scores.sql +0 -51
- package/lib/intake/postgres-queue.mjs +0 -240
- package/lib/reconcile/postgres-namespace.mjs +0 -102
- package/lib/services/local-postgres.mjs +0 -15
- package/lib/storage/backup.mjs +0 -347
- package/lib/storage/migrations.mjs +0 -187
- package/lib/storage/postgres-backup.mjs +0 -124
- package/lib/storage/sql-store.mjs +0 -45
- package/lib/storage/store-version.mjs +0 -115
- package/lib/storage/unified-storage.mjs +0 -550
- package/lib/storage/vector-store.mjs +0 -100
package/lib/storage/backup.mjs
DELETED
|
@@ -1,347 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* lib/storage/backup.mjs — full Construct backup and restore.
|
|
3
|
-
*
|
|
4
|
-
* A backup archive is a gzip-compressed tar containing:
|
|
5
|
-
*
|
|
6
|
-
* manifest.json — version, timestamp, contents list, checksums
|
|
7
|
-
* postgres/ — pg_dump output (if Postgres is reachable)
|
|
8
|
-
* observations/ — ~/.cx/observations/*.json
|
|
9
|
-
* sessions/ — ~/.cx/sessions/*.json
|
|
10
|
-
* config.env — ~/.construct/config.env (secrets stripped unless opts.includeSecrets)
|
|
11
|
-
* registry.json — specialists/registry.json snapshot
|
|
12
|
-
*
|
|
13
|
-
* The manifest uses SHA-256 checksums over each included file so
|
|
14
|
-
* `backup verify` can detect corruption without extracting.
|
|
15
|
-
*
|
|
16
|
-
* Restore is atomic: files are written to a temp directory first,
|
|
17
|
-
* then moved into place in one pass. Postgres restore is the last step
|
|
18
|
-
* because it is the hardest to roll back.
|
|
19
|
-
*/
|
|
20
|
-
|
|
21
|
-
import fs from 'node:fs';
|
|
22
|
-
import os from 'node:os';
|
|
23
|
-
import path from 'node:path';
|
|
24
|
-
import crypto from 'node:crypto';
|
|
25
|
-
import { execFileSync } from 'node:child_process';
|
|
26
|
-
|
|
27
|
-
const HOME = os.homedir();
|
|
28
|
-
const CX_DIR = path.join(HOME, '.cx');
|
|
29
|
-
const CONSTRUCT_DIR = path.join(HOME, '.construct');
|
|
30
|
-
const BACKUP_DIR = path.join(CONSTRUCT_DIR, 'backups');
|
|
31
|
-
|
|
32
|
-
// Secret env var patterns to redact in config.env.
|
|
33
|
-
const SECRET_KEYS = [
|
|
34
|
-
/TOKEN/i, /PASSWORD/i, /SECRET/i, /KEY/i, /CREDENTIAL/i,
|
|
35
|
-
/ANTHROPIC_API/i, /GITHUB_TOKEN/i, /SLACK_BOT/i, /SALESFORCE_ACCESS/i,
|
|
36
|
-
/JIRA_API/i, /LINEAR_API/i,
|
|
37
|
-
];
|
|
38
|
-
|
|
39
|
-
function isSecretKey(key) {
|
|
40
|
-
return SECRET_KEYS.some((re) => re.test(key));
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
function redactConfigEnv(content) {
|
|
44
|
-
return content.split('\n').map((line) => {
|
|
45
|
-
const trimmed = line.trim();
|
|
46
|
-
if (!trimmed || trimmed.startsWith('#')) return line;
|
|
47
|
-
const eq = trimmed.indexOf('=');
|
|
48
|
-
if (eq === -1) return line;
|
|
49
|
-
const key = trimmed.slice(0, eq).trim();
|
|
50
|
-
if (isSecretKey(key)) return `${key}=<redacted>`;
|
|
51
|
-
return line;
|
|
52
|
-
}).join('\n');
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
function sha256File(filePath) {
|
|
56
|
-
const hash = crypto.createHash('sha256');
|
|
57
|
-
hash.update(fs.readFileSync(filePath));
|
|
58
|
-
return hash.digest('hex');
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
function sha256Buffer(buf) {
|
|
62
|
-
return crypto.createHash('sha256').update(buf).digest('hex');
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function archiveName() {
|
|
66
|
-
const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
67
|
-
return `construct-backup-${ts}.tar.gz`;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
/**
|
|
71
|
-
* Create a full backup archive.
|
|
72
|
-
*
|
|
73
|
-
* @param {object} [opts]
|
|
74
|
-
* @param {boolean} [opts.includeSecrets] - include raw secret values in config.env
|
|
75
|
-
* @param {string} [opts.registryPath] - path to specialists/registry.json
|
|
76
|
-
* @param {string} [opts.destDir] - destination directory (default: ~/.construct/backups)
|
|
77
|
-
* @returns {{ path: string, manifest: object }}
|
|
78
|
-
*/
|
|
79
|
-
export async function createBackup({ includeSecrets = false, registryPath, destDir } = {}) {
|
|
80
|
-
const dest = destDir || BACKUP_DIR;
|
|
81
|
-
fs.mkdirSync(dest, { recursive: true });
|
|
82
|
-
|
|
83
|
-
const stageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'construct-backup-'));
|
|
84
|
-
const files = [];
|
|
85
|
-
const checksums = {};
|
|
86
|
-
|
|
87
|
-
try {
|
|
88
|
-
// ── Observations ──────────────────────────────────────────────────────
|
|
89
|
-
const obsDir = path.join(CX_DIR, 'observations');
|
|
90
|
-
const stagObs = path.join(stageDir, 'observations');
|
|
91
|
-
if (fs.existsSync(obsDir)) {
|
|
92
|
-
fs.mkdirSync(stagObs, { recursive: true });
|
|
93
|
-
for (const f of fs.readdirSync(obsDir).filter((n) => n.endsWith('.json'))) {
|
|
94
|
-
const src = path.join(obsDir, f);
|
|
95
|
-
const dst = path.join(stagObs, f);
|
|
96
|
-
fs.copyFileSync(src, dst);
|
|
97
|
-
checksums[`observations/${f}`] = sha256File(dst);
|
|
98
|
-
files.push(`observations/${f}`);
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
// ── Sessions ──────────────────────────────────────────────────────────
|
|
103
|
-
const sessDir = path.join(CX_DIR, 'sessions');
|
|
104
|
-
const stagSess = path.join(stageDir, 'sessions');
|
|
105
|
-
if (fs.existsSync(sessDir)) {
|
|
106
|
-
fs.mkdirSync(stagSess, { recursive: true });
|
|
107
|
-
for (const f of fs.readdirSync(sessDir).filter((n) => n.endsWith('.json'))) {
|
|
108
|
-
const src = path.join(sessDir, f);
|
|
109
|
-
const dst = path.join(stagSess, f);
|
|
110
|
-
fs.copyFileSync(src, dst);
|
|
111
|
-
checksums[`sessions/${f}`] = sha256File(dst);
|
|
112
|
-
files.push(`sessions/${f}`);
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
// ── config.env ────────────────────────────────────────────────────────
|
|
117
|
-
const configEnv = path.join(CONSTRUCT_DIR, 'config.env');
|
|
118
|
-
if (fs.existsSync(configEnv)) {
|
|
119
|
-
let content = fs.readFileSync(configEnv, 'utf8');
|
|
120
|
-
if (!includeSecrets) content = redactConfigEnv(content);
|
|
121
|
-
const dst = path.join(stageDir, 'config.env');
|
|
122
|
-
fs.writeFileSync(dst, content);
|
|
123
|
-
checksums['config.env'] = sha256Buffer(Buffer.from(content));
|
|
124
|
-
files.push('config.env');
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
// ── Registry snapshot ─────────────────────────────────────────────────
|
|
128
|
-
const regPath = registryPath || path.join(process.cwd(), 'specialists', 'registry.json');
|
|
129
|
-
if (fs.existsSync(regPath)) {
|
|
130
|
-
const dst = path.join(stageDir, 'registry.json');
|
|
131
|
-
fs.copyFileSync(regPath, dst);
|
|
132
|
-
checksums['registry.json'] = sha256File(dst);
|
|
133
|
-
files.push('registry.json');
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
// ── Postgres dump ─────────────────────────────────────────────────────
|
|
137
|
-
const pgResult = tryPostgresDump(stageDir);
|
|
138
|
-
if (pgResult.success) {
|
|
139
|
-
checksums['postgres/dump.sql'] = sha256File(path.join(stageDir, 'postgres', 'dump.sql'));
|
|
140
|
-
files.push('postgres/dump.sql');
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
// ── Manifest ──────────────────────────────────────────────────────────
|
|
144
|
-
const manifest = {
|
|
145
|
-
version: 1,
|
|
146
|
-
createdAt: new Date().toISOString(),
|
|
147
|
-
includesSecrets: includeSecrets,
|
|
148
|
-
contents: files,
|
|
149
|
-
checksums,
|
|
150
|
-
};
|
|
151
|
-
fs.writeFileSync(path.join(stageDir, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
|
152
|
-
|
|
153
|
-
// ── Pack to tar.gz ────────────────────────────────────────────────────
|
|
154
|
-
const outPath = path.join(dest, archiveName());
|
|
155
|
-
execFileSync('tar', ['-czf', outPath, '-C', stageDir, '.'], { stdio: 'pipe' });
|
|
156
|
-
|
|
157
|
-
return { path: outPath, manifest };
|
|
158
|
-
} finally {
|
|
159
|
-
fs.rmSync(stageDir, { recursive: true, force: true });
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
function tryPostgresDump(stageDir) {
|
|
164
|
-
const dumpDir = path.join(stageDir, 'postgres');
|
|
165
|
-
const dumpFile = path.join(dumpDir, 'dump.sql');
|
|
166
|
-
|
|
167
|
-
const dbUrl = process.env.DATABASE_URL || process.env.CONSTRUCT_DATABASE_URL;
|
|
168
|
-
if (!dbUrl) return { success: false, reason: 'no DATABASE_URL' };
|
|
169
|
-
|
|
170
|
-
try {
|
|
171
|
-
fs.mkdirSync(dumpDir, { recursive: true });
|
|
172
|
-
execFileSync('pg_dump', [dbUrl, '--no-password', '-f', dumpFile], {
|
|
173
|
-
stdio: ['ignore', 'pipe', 'pipe'],
|
|
174
|
-
timeout: 60_000,
|
|
175
|
-
});
|
|
176
|
-
return { success: true };
|
|
177
|
-
} catch (err) {
|
|
178
|
-
return { success: false, reason: err.message };
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
/**
|
|
183
|
-
* Verify a backup archive by comparing stored checksums to actual file hashes.
|
|
184
|
-
*
|
|
185
|
-
* @param {string} archivePath
|
|
186
|
-
* @returns {{ ok: boolean, errors: string[] }}
|
|
187
|
-
*/
|
|
188
|
-
export async function verifyBackup(archivePath) {
|
|
189
|
-
const extractDir = fs.mkdtempSync(path.join(os.tmpdir(), 'construct-verify-'));
|
|
190
|
-
const errors = [];
|
|
191
|
-
try {
|
|
192
|
-
execFileSync('tar', ['-xzf', archivePath, '-C', extractDir], { stdio: 'pipe' });
|
|
193
|
-
|
|
194
|
-
const manifestPath = path.join(extractDir, 'manifest.json');
|
|
195
|
-
if (!fs.existsSync(manifestPath)) {
|
|
196
|
-
return { ok: false, errors: ['manifest.json missing'] };
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
200
|
-
for (const [rel, expected] of Object.entries(manifest.checksums || {})) {
|
|
201
|
-
const abs = path.join(extractDir, rel);
|
|
202
|
-
if (!fs.existsSync(abs)) {
|
|
203
|
-
errors.push(`missing: ${rel}`);
|
|
204
|
-
continue;
|
|
205
|
-
}
|
|
206
|
-
const actual = sha256File(abs);
|
|
207
|
-
if (actual !== expected) errors.push(`checksum mismatch: ${rel}`);
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
return { ok: errors.length === 0, errors };
|
|
211
|
-
} finally {
|
|
212
|
-
fs.rmSync(extractDir, { recursive: true, force: true });
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
/**
|
|
217
|
-
* Restore a backup archive. Prompts for confirmation before writing.
|
|
218
|
-
*
|
|
219
|
-
* @param {string} archivePath
|
|
220
|
-
* @param {object} [opts]
|
|
221
|
-
* @param {boolean} [opts.yes] - skip confirmation prompt
|
|
222
|
-
* @returns {{ ok: boolean, restored: string[], errors: string[] }}
|
|
223
|
-
*/
|
|
224
|
-
export async function restoreBackup(archivePath, { yes = false } = {}) {
|
|
225
|
-
const verify = await verifyBackup(archivePath);
|
|
226
|
-
if (!verify.ok) {
|
|
227
|
-
return { ok: false, restored: [], errors: [`verification failed: ${verify.errors.join(', ')}`] };
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
const extractDir = fs.mkdtempSync(path.join(os.tmpdir(), 'construct-restore-'));
|
|
231
|
-
const restored = [];
|
|
232
|
-
const errors = [];
|
|
233
|
-
|
|
234
|
-
try {
|
|
235
|
-
execFileSync('tar', ['-xzf', archivePath, '-C', extractDir], { stdio: 'pipe' });
|
|
236
|
-
|
|
237
|
-
const manifest = JSON.parse(fs.readFileSync(path.join(extractDir, 'manifest.json'), 'utf8'));
|
|
238
|
-
|
|
239
|
-
// Restore observations
|
|
240
|
-
const stagObs = path.join(extractDir, 'observations');
|
|
241
|
-
if (fs.existsSync(stagObs)) {
|
|
242
|
-
const dest = path.join(CX_DIR, 'observations');
|
|
243
|
-
fs.mkdirSync(dest, { recursive: true });
|
|
244
|
-
for (const f of fs.readdirSync(stagObs)) {
|
|
245
|
-
fs.copyFileSync(path.join(stagObs, f), path.join(dest, f));
|
|
246
|
-
restored.push(`observations/${f}`);
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
// Restore sessions
|
|
251
|
-
const stagSess = path.join(extractDir, 'sessions');
|
|
252
|
-
if (fs.existsSync(stagSess)) {
|
|
253
|
-
const dest = path.join(CX_DIR, 'sessions');
|
|
254
|
-
fs.mkdirSync(dest, { recursive: true });
|
|
255
|
-
for (const f of fs.readdirSync(stagSess)) {
|
|
256
|
-
fs.copyFileSync(path.join(stagSess, f), path.join(dest, f));
|
|
257
|
-
restored.push(`sessions/${f}`);
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
// Restore config.env — only if it contained no redacted lines
|
|
262
|
-
const stagConfig = path.join(extractDir, 'config.env');
|
|
263
|
-
if (fs.existsSync(stagConfig)) {
|
|
264
|
-
const content = fs.readFileSync(stagConfig, 'utf8');
|
|
265
|
-
if (!content.includes('<redacted>')) {
|
|
266
|
-
fs.mkdirSync(CONSTRUCT_DIR, { recursive: true });
|
|
267
|
-
fs.writeFileSync(path.join(CONSTRUCT_DIR, 'config.env'), content, { mode: 0o600 });
|
|
268
|
-
restored.push('config.env');
|
|
269
|
-
} else {
|
|
270
|
-
errors.push('config.env has redacted secrets — restore credentials manually');
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
// Restore registry.json
|
|
275
|
-
const stagReg = path.join(extractDir, 'registry.json');
|
|
276
|
-
if (fs.existsSync(stagReg)) {
|
|
277
|
-
const dest = path.join(process.cwd(), 'specialists', 'registry.json');
|
|
278
|
-
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
279
|
-
fs.copyFileSync(stagReg, dest);
|
|
280
|
-
restored.push('registry.json');
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
// Restore Postgres (last — hardest to undo)
|
|
284
|
-
const stagPg = path.join(extractDir, 'postgres', 'dump.sql');
|
|
285
|
-
if (fs.existsSync(stagPg)) {
|
|
286
|
-
const pgResult = tryPostgresRestore(stagPg);
|
|
287
|
-
if (pgResult.success) {
|
|
288
|
-
restored.push('postgres');
|
|
289
|
-
} else {
|
|
290
|
-
errors.push(`postgres restore failed: ${pgResult.reason}`);
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
return { ok: errors.length === 0, restored, errors };
|
|
295
|
-
} finally {
|
|
296
|
-
fs.rmSync(extractDir, { recursive: true, force: true });
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
function tryPostgresRestore(dumpFile) {
|
|
301
|
-
const dbUrl = process.env.DATABASE_URL || process.env.CONSTRUCT_DATABASE_URL;
|
|
302
|
-
if (!dbUrl) return { success: false, reason: 'no DATABASE_URL' };
|
|
303
|
-
try {
|
|
304
|
-
execFileSync('psql', [dbUrl, '-f', dumpFile], {
|
|
305
|
-
stdio: ['ignore', 'pipe', 'pipe'],
|
|
306
|
-
timeout: 120_000,
|
|
307
|
-
});
|
|
308
|
-
return { success: true };
|
|
309
|
-
} catch (err) {
|
|
310
|
-
return { success: false, reason: err.message };
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
/**
|
|
315
|
-
* List backup archives in the default backup directory.
|
|
316
|
-
*/
|
|
317
|
-
export function listBackups(dir = BACKUP_DIR) {
|
|
318
|
-
if (!fs.existsSync(dir)) return [];
|
|
319
|
-
return fs.readdirSync(dir)
|
|
320
|
-
.filter((f) => f.startsWith('construct-backup-') && f.endsWith('.tar.gz'))
|
|
321
|
-
.sort()
|
|
322
|
-
.reverse()
|
|
323
|
-
.map((f) => {
|
|
324
|
-
const fullPath = path.join(dir, f);
|
|
325
|
-
const stat = fs.statSync(fullPath);
|
|
326
|
-
return { name: f, path: fullPath, size: stat.size, mtime: stat.mtime };
|
|
327
|
-
});
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
// Keep the `keep` most recent backups in `dir`; unlink the rest. The return
|
|
331
|
-
// value names each pruned file by archive basename. Per-file unlink errors
|
|
332
|
-
// are swallowed so one locked archive can't abort the whole pass.
|
|
333
|
-
export function pruneBackups({ keep = 10, dir = BACKUP_DIR } = {}) {
|
|
334
|
-
const list = listBackups(dir);
|
|
335
|
-
if (list.length <= keep) return { kept: list.length, removed: [] };
|
|
336
|
-
const toRemove = list.slice(keep);
|
|
337
|
-
const removed = [];
|
|
338
|
-
for (const b of toRemove) {
|
|
339
|
-
try {
|
|
340
|
-
fs.unlinkSync(b.path);
|
|
341
|
-
removed.push(b.name);
|
|
342
|
-
} catch {
|
|
343
|
-
/* best effort — leave file in place if it's locked or unreadable */
|
|
344
|
-
}
|
|
345
|
-
}
|
|
346
|
-
return { kept: keep, removed };
|
|
347
|
-
}
|
|
@@ -1,187 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* lib/storage/migrations.mjs — Postgres migration runner with hash tracking.
|
|
3
|
-
*
|
|
4
|
-
* Walks `db/schema/*.sql` in lexical order and applies any not yet
|
|
5
|
-
* recorded in `construct_schema_migrations`. Each applied file's SHA256 is
|
|
6
|
-
* stored alongside the filename. Re-running the runner against an unchanged
|
|
7
|
-
* tree is a no-op.
|
|
8
|
-
*
|
|
9
|
-
* Drift detection: if an applied file's SHA changes between runs, the runner
|
|
10
|
-
* fails fast with a precise error. SQL files are append-only by convention —
|
|
11
|
-
* to evolve schema after a migration ships, write a new file with a higher
|
|
12
|
-
* sequence number.
|
|
13
|
-
*
|
|
14
|
-
* Repair mode: long-lived developer databases pre-date the append-only policy
|
|
15
|
-
* and carry stale SHAs for files that have since evolved. Call runMigrations
|
|
16
|
-
* with `repair: true` (or `construct storage repair-migrations`) to re-apply
|
|
17
|
-
* each drifted file. The repair refuses on any file containing a destructive
|
|
18
|
-
* statement (`DROP`, `TRUNCATE`, `ALTER … DROP`) so the safety bar stays
|
|
19
|
-
* "only fully-idempotent migrations may be re-recorded."
|
|
20
|
-
*
|
|
21
|
-
* The bookkeeping table (`construct_schema_migrations`) is created inline
|
|
22
|
-
* before applied state is checked, so the runner self-bootstraps and a fresh
|
|
23
|
-
* database can run from any starting migration without a prerequisite file.
|
|
24
|
-
*/
|
|
25
|
-
|
|
26
|
-
import { readdirSync, readFileSync } from 'node:fs';
|
|
27
|
-
import { join, dirname } from 'node:path';
|
|
28
|
-
import { fileURLToPath } from 'node:url';
|
|
29
|
-
import { createHash } from 'node:crypto';
|
|
30
|
-
|
|
31
|
-
const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
|
|
32
|
-
const DEFAULT_MIGRATIONS_DIR = join(MODULE_DIR, '..', '..', 'db', 'schema');
|
|
33
|
-
|
|
34
|
-
function sha256(text) {
|
|
35
|
-
return createHash('sha256').update(text).digest('hex');
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
function listMigrationFiles(dir) {
|
|
39
|
-
return readdirSync(dir)
|
|
40
|
-
.filter((name) => name.endsWith('.sql'))
|
|
41
|
-
.sort();
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
const BOOKKEEPING_DDL = `
|
|
45
|
-
CREATE TABLE IF NOT EXISTS construct_schema_migrations (
|
|
46
|
-
filename TEXT PRIMARY KEY,
|
|
47
|
-
sha TEXT NOT NULL,
|
|
48
|
-
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
49
|
-
);
|
|
50
|
-
`;
|
|
51
|
-
|
|
52
|
-
// Heuristic: a migration is safe to re-apply (idempotent) when it contains no
|
|
53
|
-
// destructive statements. Comments are stripped first so a `-- DROP …` note in
|
|
54
|
-
// a docstring does not falsely flag the file.
|
|
55
|
-
const DESTRUCTIVE_RE = /\b(drop\s+(table|index|view|column|schema|extension|function|sequence)\b|truncate\b|alter\s+table\s+[^;]*?\bdrop\b|delete\s+from\b)/i;
|
|
56
|
-
|
|
57
|
-
function stripSqlComments(sql) {
|
|
58
|
-
return sql
|
|
59
|
-
.replace(/--[^\n]*/g, '')
|
|
60
|
-
.replace(/\/\*[\s\S]*?\*\//g, '');
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
export function isIdempotentMigration(sql) {
|
|
64
|
-
return !DESTRUCTIVE_RE.test(stripSqlComments(sql));
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Apply pending migrations. Returns a per-file outcome summary.
|
|
69
|
-
*
|
|
70
|
-
* @param {object} client — postgres client (from createSqlClient)
|
|
71
|
-
* @param {object} [opts]
|
|
72
|
-
* @param {string} [opts.dir] — schema/migrations directory, defaults to db/schema
|
|
73
|
-
* @param {boolean} [opts.repair] — when true, re-apply drifted files that are
|
|
74
|
-
* verifiably idempotent and update their recorded SHA. Destructive drift
|
|
75
|
-
* still throws. Default false.
|
|
76
|
-
* @returns {Promise<{ applied: string[], skipped: string[], repaired: string[], drift: Array<{ filename: string, expected: string, actual: string, idempotent: boolean }> }>}
|
|
77
|
-
*/
|
|
78
|
-
export async function runMigrations(client, { dir = DEFAULT_MIGRATIONS_DIR, repair = false } = {}) {
|
|
79
|
-
if (!client) throw new Error('runMigrations requires a postgres client');
|
|
80
|
-
|
|
81
|
-
await client.unsafe(BOOKKEEPING_DDL);
|
|
82
|
-
|
|
83
|
-
const recorded = await client`
|
|
84
|
-
SELECT filename, sha FROM construct_schema_migrations
|
|
85
|
-
`;
|
|
86
|
-
const recordedMap = new Map(recorded.map((r) => [r.filename, r.sha]));
|
|
87
|
-
|
|
88
|
-
const files = listMigrationFiles(dir);
|
|
89
|
-
const applied = [];
|
|
90
|
-
const skipped = [];
|
|
91
|
-
const repaired = [];
|
|
92
|
-
const drift = [];
|
|
93
|
-
|
|
94
|
-
for (const filename of files) {
|
|
95
|
-
const path = join(dir, filename);
|
|
96
|
-
const contents = readFileSync(path, 'utf8');
|
|
97
|
-
const sha = sha256(contents);
|
|
98
|
-
|
|
99
|
-
const prev = recordedMap.get(filename);
|
|
100
|
-
if (prev) {
|
|
101
|
-
if (prev !== sha) {
|
|
102
|
-
const idempotent = isIdempotentMigration(contents);
|
|
103
|
-
if (repair && idempotent) {
|
|
104
|
-
// Re-apply the migration (its IF NOT EXISTS clauses make this a
|
|
105
|
-
// no-op against the live schema) and update the recorded SHA.
|
|
106
|
-
await client.begin(async (tx) => {
|
|
107
|
-
await tx.unsafe(contents);
|
|
108
|
-
await tx`
|
|
109
|
-
UPDATE construct_schema_migrations
|
|
110
|
-
SET sha = ${sha}, applied_at = now()
|
|
111
|
-
WHERE filename = ${filename}
|
|
112
|
-
`;
|
|
113
|
-
});
|
|
114
|
-
repaired.push(filename);
|
|
115
|
-
} else {
|
|
116
|
-
drift.push({ filename, expected: prev, actual: sha, idempotent });
|
|
117
|
-
}
|
|
118
|
-
} else {
|
|
119
|
-
skipped.push(filename);
|
|
120
|
-
}
|
|
121
|
-
continue;
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
await client.begin(async (tx) => {
|
|
125
|
-
await tx.unsafe(contents);
|
|
126
|
-
await tx`
|
|
127
|
-
INSERT INTO construct_schema_migrations (filename, sha)
|
|
128
|
-
VALUES (${filename}, ${sha})
|
|
129
|
-
ON CONFLICT (filename) DO UPDATE SET sha = EXCLUDED.sha, applied_at = now()
|
|
130
|
-
`;
|
|
131
|
-
});
|
|
132
|
-
applied.push(filename);
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
if (drift.length > 0) {
|
|
136
|
-
const detail = drift
|
|
137
|
-
.map((d) => ` ${d.filename}: applied SHA ${d.expected.slice(0, 12)}…, on-disk SHA ${d.actual.slice(0, 12)}…${d.idempotent ? ' (idempotent — repairable via `construct storage repair-migrations`)' : ' (NOT idempotent — write a new migration file)'}`)
|
|
138
|
-
.join('\n');
|
|
139
|
-
throw new Error(
|
|
140
|
-
`Migration drift detected (a previously-applied migration file has changed). ` +
|
|
141
|
-
`Migrations are append-only — write a new file with a higher sequence number ` +
|
|
142
|
-
`to evolve the schema. Drift:\n${detail}`
|
|
143
|
-
);
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
return { applied, skipped, repaired, drift };
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
/**
|
|
150
|
-
* Diagnostic for `construct doctor`: returns last applied filename, count,
|
|
151
|
-
* and any drift, without throwing.
|
|
152
|
-
*/
|
|
153
|
-
export async function describeMigrations(client, { dir = DEFAULT_MIGRATIONS_DIR } = {}) {
|
|
154
|
-
if (!client) return { ok: false, reason: 'no_sql_client' };
|
|
155
|
-
try {
|
|
156
|
-
await client.unsafe(BOOKKEEPING_DDL);
|
|
157
|
-
const rows = await client`
|
|
158
|
-
SELECT filename, sha, applied_at
|
|
159
|
-
FROM construct_schema_migrations
|
|
160
|
-
ORDER BY applied_at DESC, filename DESC
|
|
161
|
-
LIMIT 1
|
|
162
|
-
`;
|
|
163
|
-
const onDisk = listMigrationFiles(dir);
|
|
164
|
-
const applied = await client`SELECT count(*)::int AS n FROM construct_schema_migrations`;
|
|
165
|
-
const recorded = await client`SELECT filename, sha FROM construct_schema_migrations`;
|
|
166
|
-
const recordedMap = new Map(recorded.map((r) => [r.filename, r.sha]));
|
|
167
|
-
const drift = [];
|
|
168
|
-
for (const filename of onDisk) {
|
|
169
|
-
const contents = readFileSync(join(dir, filename), 'utf8');
|
|
170
|
-
const sha = sha256(contents);
|
|
171
|
-
const prev = recordedMap.get(filename);
|
|
172
|
-
if (prev && prev !== sha) {
|
|
173
|
-
drift.push({ filename, idempotent: isIdempotentMigration(contents) });
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
return {
|
|
177
|
-
ok: true,
|
|
178
|
-
lastApplied: rows[0]?.filename || null,
|
|
179
|
-
lastAppliedAt: rows[0]?.applied_at || null,
|
|
180
|
-
appliedCount: applied[0]?.n ?? 0,
|
|
181
|
-
onDiskCount: onDisk.length,
|
|
182
|
-
drift,
|
|
183
|
-
};
|
|
184
|
-
} catch (err) {
|
|
185
|
-
return { ok: false, reason: err?.message || 'describe-migrations failed' };
|
|
186
|
-
}
|
|
187
|
-
}
|
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* lib/storage/postgres-backup.mjs — durable stash/restore for the managed construct-postgres container.
|
|
4
|
-
*
|
|
5
|
-
* Mirrors the telemetry backup pattern in service-manager.mjs. Entry points:
|
|
6
|
-
* - construct down → stashConstructDb (dump before container stop)
|
|
7
|
-
* - construct init → restoreConstructDb (reload after fresh container start)
|
|
8
|
-
*
|
|
9
|
-
* Dumps are stored in ~/.construct/backups/postgres/ as pg_dump custom-format
|
|
10
|
-
* files. The N most recent are kept; older ones are pruned automatically.
|
|
11
|
-
*/
|
|
12
|
-
import fs from 'node:fs';
|
|
13
|
-
import os from 'node:os';
|
|
14
|
-
import path from 'node:path';
|
|
15
|
-
import { spawnSync } from 'node:child_process';
|
|
16
|
-
|
|
17
|
-
const POSTGRES_CONTAINER = 'construct-postgres';
|
|
18
|
-
const POSTGRES_USER = 'construct';
|
|
19
|
-
const POSTGRES_DB = 'construct';
|
|
20
|
-
const DEFAULT_KEEP = 5;
|
|
21
|
-
|
|
22
|
-
function pruneBackupDir(dir, keep) {
|
|
23
|
-
try {
|
|
24
|
-
const dumps = fs.readdirSync(dir)
|
|
25
|
-
.filter((f) => f.startsWith('construct-') && f.endsWith('.dump'))
|
|
26
|
-
.sort()
|
|
27
|
-
.reverse();
|
|
28
|
-
for (const dump of dumps.slice(keep)) {
|
|
29
|
-
fs.rmSync(path.join(dir, dump), { force: true });
|
|
30
|
-
fs.rmSync(path.join(dir, dump.replace('.dump', '.json')), { force: true });
|
|
31
|
-
}
|
|
32
|
-
} catch { /* non-critical */ }
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* Dump the construct Postgres database to a timestamped file under
|
|
37
|
-
* ~/.construct/backups/postgres/. Safe to call when the container is not
|
|
38
|
-
* running — returns { status: 'no-data' } silently.
|
|
39
|
-
*/
|
|
40
|
-
export function stashConstructDb({
|
|
41
|
-
homeDir = os.homedir(),
|
|
42
|
-
spawnSyncFn = spawnSync,
|
|
43
|
-
keep = DEFAULT_KEEP,
|
|
44
|
-
} = {}) {
|
|
45
|
-
const stashDir = path.join(homeDir, '.construct', 'backups', 'postgres');
|
|
46
|
-
fs.mkdirSync(stashDir, { recursive: true });
|
|
47
|
-
|
|
48
|
-
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
49
|
-
const dumpFile = path.join(stashDir, `construct-${timestamp}.dump`);
|
|
50
|
-
const manifestFile = path.join(stashDir, `construct-${timestamp}.json`);
|
|
51
|
-
|
|
52
|
-
const dump = spawnSyncFn('docker', [
|
|
53
|
-
'exec', POSTGRES_CONTAINER,
|
|
54
|
-
'pg_dump', '-U', POSTGRES_USER, '-d', POSTGRES_DB,
|
|
55
|
-
'-Fc', '--data-only',
|
|
56
|
-
], { stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 200 * 1024 * 1024 });
|
|
57
|
-
|
|
58
|
-
const hasData = dump.status === 0 && dump.stdout?.length > 100;
|
|
59
|
-
if (!hasData) return { status: 'no-data', stashPath: null };
|
|
60
|
-
|
|
61
|
-
fs.writeFileSync(dumpFile, dump.stdout);
|
|
62
|
-
fs.writeFileSync(manifestFile, JSON.stringify({
|
|
63
|
-
version: 1,
|
|
64
|
-
createdAt: new Date().toISOString(),
|
|
65
|
-
reason: 'pre-shutdown-stash',
|
|
66
|
-
dumpFile: path.basename(dumpFile),
|
|
67
|
-
dumpBytes: dump.stdout.length,
|
|
68
|
-
}, null, 2) + '\n');
|
|
69
|
-
|
|
70
|
-
pruneBackupDir(stashDir, keep);
|
|
71
|
-
return { status: 'ok', stashPath: dumpFile, bytes: dump.stdout.length };
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
/**
|
|
75
|
-
* Restore the most recent stash into a running construct-postgres container.
|
|
76
|
-
* Safe to call when no stash exists — returns { status: 'no-stash' } silently.
|
|
77
|
-
* The schema must already exist (run migration before calling this).
|
|
78
|
-
*/
|
|
79
|
-
export function restoreConstructDb({
|
|
80
|
-
homeDir = os.homedir(),
|
|
81
|
-
spawnSyncFn = spawnSync,
|
|
82
|
-
} = {}) {
|
|
83
|
-
const stashDir = path.join(homeDir, '.construct', 'backups', 'postgres');
|
|
84
|
-
if (!fs.existsSync(stashDir)) return { status: 'no-stash' };
|
|
85
|
-
|
|
86
|
-
const dumps = fs.readdirSync(stashDir)
|
|
87
|
-
.filter((f) => f.startsWith('construct-') && f.endsWith('.dump'))
|
|
88
|
-
.sort()
|
|
89
|
-
.reverse();
|
|
90
|
-
if (dumps.length === 0) return { status: 'no-stash' };
|
|
91
|
-
|
|
92
|
-
const latestDump = path.join(stashDir, dumps[0]);
|
|
93
|
-
|
|
94
|
-
const cp = spawnSyncFn('docker', [
|
|
95
|
-
'cp', latestDump, `${POSTGRES_CONTAINER}:/tmp/construct.dump`,
|
|
96
|
-
], { stdio: 'ignore' });
|
|
97
|
-
if (cp.status !== 0) return { status: 'copy-failed', stashPath: latestDump };
|
|
98
|
-
|
|
99
|
-
const restore = spawnSyncFn('docker', [
|
|
100
|
-
'exec', POSTGRES_CONTAINER,
|
|
101
|
-
'pg_restore', '-U', POSTGRES_USER, '-d', POSTGRES_DB,
|
|
102
|
-
'--data-only', '--disable-triggers', '--no-owner',
|
|
103
|
-
'--if-exists',
|
|
104
|
-
'/tmp/construct.dump',
|
|
105
|
-
], { stdio: 'ignore' });
|
|
106
|
-
|
|
107
|
-
return {
|
|
108
|
-
status: restore.status === 0 ? 'restored' : 'restore-failed',
|
|
109
|
-
stashPath: latestDump,
|
|
110
|
-
exitCode: restore.status,
|
|
111
|
-
};
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
/**
|
|
115
|
-
* Delete all stashed backups under ~/.construct/backups/postgres/.
|
|
116
|
-
* Called by `construct storage reset` and uninstall flows.
|
|
117
|
-
*/
|
|
118
|
-
export function purgeConstructDbStashes({ homeDir = os.homedir() } = {}) {
|
|
119
|
-
const stashDir = path.join(homeDir, '.construct', 'backups', 'postgres');
|
|
120
|
-
if (!fs.existsSync(stashDir)) return { status: 'ok', deletedCount: 0 };
|
|
121
|
-
const files = fs.readdirSync(stashDir).map((f) => path.join(stashDir, f));
|
|
122
|
-
for (const f of files) fs.rmSync(f, { force: true });
|
|
123
|
-
return { status: 'ok', deletedCount: files.length };
|
|
124
|
-
}
|