@pgpmjs/core 7.14.0 → 7.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apply/apply-spec.js +84 -9
- package/apply/index.d.ts +1 -0
- package/apply/index.js +1 -0
- package/apply/materialize.js +15 -2
- package/apply/reuse.d.ts +52 -0
- package/apply/reuse.js +226 -0
- package/apply/types.d.ts +89 -0
- package/bundle/artifact.d.ts +35 -0
- package/bundle/artifact.js +96 -0
- package/bundle/deploy-bundled.d.ts +71 -0
- package/bundle/deploy-bundled.js +177 -0
- package/bundle/index.d.ts +2 -0
- package/bundle/index.js +4 -0
- package/core/class/pgpm.js +39 -54
- package/esm/apply/apply-spec.js +84 -9
- package/esm/apply/index.js +1 -0
- package/esm/apply/materialize.js +15 -2
- package/esm/apply/reuse.js +221 -0
- package/esm/bundle/artifact.js +89 -0
- package/esm/bundle/deploy-bundled.js +170 -0
- package/esm/bundle/index.js +4 -0
- package/esm/core/class/pgpm.js +39 -54
- package/esm/packaging/package.js +8 -1
- package/esm/projects/deploy.js +17 -54
- package/package.json +10 -10
- package/packaging/package.d.ts +7 -1
- package/packaging/package.js +8 -1
- package/projects/deploy.js +17 -54
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import { mkdtempSync } from 'fs';
|
|
2
|
+
import { tmpdir } from 'os';
|
|
3
|
+
import { join } from 'path';
|
|
4
|
+
import { hashString } from '@pgpmjs/ast';
|
|
5
|
+
import { bundleFromModule, materializeBundle, splitBundle, transpileBundle, verifyBundle } from '@pgpmjs/bundle';
|
|
6
|
+
import { partitionModule } from '@pgpmjs/slice';
|
|
7
|
+
import { loadModule, makeSchemaTranspiler } from '@pgpmjs/transform';
|
|
8
|
+
/** Whether an apply spec is a reuse (shared/per-tenant split) spec. */
|
|
9
|
+
export function isReuseSpec(spec) {
|
|
10
|
+
return spec.reuse !== undefined;
|
|
11
|
+
}
|
|
12
|
+
/** pgpm change-path folder → object route kind. */
|
|
13
|
+
const FOLDER_KIND = {
|
|
14
|
+
tables: 'table',
|
|
15
|
+
views: 'view',
|
|
16
|
+
functions: 'function',
|
|
17
|
+
procedures: 'procedure',
|
|
18
|
+
types: 'type'
|
|
19
|
+
};
|
|
20
|
+
// Mirrors materialize.ts: bare-schema-name literal helpers the AST pass can't
|
|
21
|
+
// reach are remapped by a narrow string pre-pass.
|
|
22
|
+
const SCHEMA_NAME_LITERAL_FUNCS = ['verify_schema', 'has_schema_privilege'];
|
|
23
|
+
const schemaNameLiteralPass = (content, schemaMapping) => {
|
|
24
|
+
const pattern = new RegExp(`\\b(${SCHEMA_NAME_LITERAL_FUNCS.join('|')})\\s*\\(\\s*'([^']+)'`, 'gi');
|
|
25
|
+
return content.replace(pattern, (match, _fn, schema) => {
|
|
26
|
+
const mapped = schemaMapping.get(schema);
|
|
27
|
+
return mapped ? match.replace(`'${schema}'`, `'${mapped}'`) : match;
|
|
28
|
+
});
|
|
29
|
+
};
|
|
30
|
+
/** Parse a `schemas/<schema>/<folder>/<name>…` change path into its object identity. */
|
|
31
|
+
function parseObjectChange(change) {
|
|
32
|
+
const parts = change.split('/');
|
|
33
|
+
if (parts[0] !== 'schemas' || parts.length < 2)
|
|
34
|
+
return null;
|
|
35
|
+
const schema = parts[1];
|
|
36
|
+
const folder = parts[2];
|
|
37
|
+
if (folder === undefined || folder === 'schema')
|
|
38
|
+
return { schema, folder: 'schema' };
|
|
39
|
+
return { schema, folder, name: parts[3] };
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Deterministic name of the shared module a reuse proxy expands into. Derived
|
|
43
|
+
* from the source module, the shared-schema mapping, and the per-tenant seed
|
|
44
|
+
* set — the exact inputs that determine the shared partition — so two tenant
|
|
45
|
+
* proxies over the same source/shared/seed configuration resolve to *one*
|
|
46
|
+
* shared module (deployed once), while any difference isolates them.
|
|
47
|
+
*/
|
|
48
|
+
export function resolveSharedModuleName(spec) {
|
|
49
|
+
if (spec.reuse.sharedName)
|
|
50
|
+
return spec.reuse.sharedName;
|
|
51
|
+
const canonical = JSON.stringify({
|
|
52
|
+
source: spec.source.module,
|
|
53
|
+
sharedSchema: Object.entries(spec.reuse.sharedSchema).sort(([a], [b]) => a.localeCompare(b)),
|
|
54
|
+
seeds: spec.reuse.perTenant
|
|
55
|
+
.map(s => `${s.fromSchema}.${s.kind}.${s.name}`)
|
|
56
|
+
.sort((a, b) => a.localeCompare(b))
|
|
57
|
+
});
|
|
58
|
+
return `${spec.source.module}-shared-${hashString(canonical).slice(0, 8)}`;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Expand a reuse proxy into two deployable modules: a shared module (objects the
|
|
62
|
+
* classifier proves tenant-independent, deployed once) and a per-tenant module
|
|
63
|
+
* (objects reachable from the per-tenant seeds, materialized per instance) that
|
|
64
|
+
* `requires` the shared one.
|
|
65
|
+
*
|
|
66
|
+
* Pipeline over existing primitives:
|
|
67
|
+
* `bundleFromModule` → `partitionModule` (classifier) →
|
|
68
|
+
* `transpileBundle(makeSchemaTranspiler)` routing shared objects to the shared
|
|
69
|
+
* schema and per-tenant objects to the per-tenant schema in *one* pass (so
|
|
70
|
+
* cross-boundary references resolve correctly) → `splitBundle` with a per-tenant
|
|
71
|
+
* schema bootstrap → `verifyBundle` → `materializeBundle`.
|
|
72
|
+
*/
|
|
73
|
+
export async function materializeReuseModule(options) {
|
|
74
|
+
const { sourceDir, spec } = options;
|
|
75
|
+
const perTenantName = spec.name;
|
|
76
|
+
const sharedName = resolveSharedModuleName(spec);
|
|
77
|
+
const reuse = spec.reuse;
|
|
78
|
+
await loadModule();
|
|
79
|
+
const source = bundleFromModule(sourceDir);
|
|
80
|
+
if (spec.source.bundleDigest && spec.source.bundleDigest !== source.manifest.digest) {
|
|
81
|
+
throw new Error(`Apply spec for "${perTenantName}" pins source bundle digest ${spec.source.bundleDigest}, ` +
|
|
82
|
+
`but the installed source "${spec.source.module}" hashes to ${source.manifest.digest}. ` +
|
|
83
|
+
`Reinstall the pinned version or update the spec.`);
|
|
84
|
+
}
|
|
85
|
+
// 1. Classify: which source changes must be per-tenant (a seed or a dependent).
|
|
86
|
+
const partition = partitionModule({
|
|
87
|
+
moduleDir: sourceDir,
|
|
88
|
+
seedObjects: reuse.perTenant.map(s => ({ schema: s.fromSchema, name: s.name }))
|
|
89
|
+
});
|
|
90
|
+
const unknownSeeds = partition.warnings.filter(w => w.kind === 'unknown-seed');
|
|
91
|
+
if (unknownSeeds.length > 0) {
|
|
92
|
+
throw new Error(`Reuse spec for "${perTenantName}": seed object(s) not found in source "${spec.source.module}": ` +
|
|
93
|
+
unknownSeeds.map(w => w.change).join(', '));
|
|
94
|
+
}
|
|
95
|
+
// 2. Object routes: every per-tenant change's object goes to the per-tenant
|
|
96
|
+
// target; everything else defaults (schemaMap) to the shared target. A
|
|
97
|
+
// single router keeps cross-boundary references correct (a per-tenant
|
|
98
|
+
// function landing in the tenant schema still references the shared helper
|
|
99
|
+
// in the shared schema).
|
|
100
|
+
const routes = [];
|
|
101
|
+
for (const change of partition.perTenant) {
|
|
102
|
+
const obj = parseObjectChange(change);
|
|
103
|
+
if (!obj || obj.folder === 'schema' || !obj.name) {
|
|
104
|
+
throw new Error(`Reuse spec for "${perTenantName}": per-tenant change "${change}" is not an object change ` +
|
|
105
|
+
`and cannot be routed to a per-tenant schema; refine the seeds so it stays shared, or ` +
|
|
106
|
+
`it names an unsupported object kind`);
|
|
107
|
+
}
|
|
108
|
+
const kind = FOLDER_KIND[obj.folder];
|
|
109
|
+
if (!kind) {
|
|
110
|
+
throw new Error(`Reuse spec for "${perTenantName}": per-tenant change "${change}" has unsupported object ` +
|
|
111
|
+
`folder "${obj.folder}"`);
|
|
112
|
+
}
|
|
113
|
+
const toSchema = spec.schemas[obj.schema];
|
|
114
|
+
if (!toSchema) {
|
|
115
|
+
throw new Error(`Reuse spec for "${perTenantName}": per-tenant object in schema "${obj.schema}" has no ` +
|
|
116
|
+
`per-tenant target in "schemas"`);
|
|
117
|
+
}
|
|
118
|
+
routes.push({ fromSchema: obj.schema, kind, name: obj.name, toSchema });
|
|
119
|
+
}
|
|
120
|
+
const assumeSchemasExist = [
|
|
121
|
+
...new Set([...Object.values(reuse.sharedSchema), ...Object.values(spec.schemas)])
|
|
122
|
+
];
|
|
123
|
+
const provenance = {
|
|
124
|
+
appliedFrom: spec.source.module,
|
|
125
|
+
...(spec.source.package ? { sourcePackage: spec.source.package } : {}),
|
|
126
|
+
...(spec.source.version ? { sourceVersion: spec.source.version } : {})
|
|
127
|
+
};
|
|
128
|
+
// 3. Transpile once: schema default = shared target, per-object overrides =
|
|
129
|
+
// per-tenant target.
|
|
130
|
+
const main = makeSchemaTranspiler({
|
|
131
|
+
schemaMap: reuse.sharedSchema,
|
|
132
|
+
routes,
|
|
133
|
+
extensions: spec.extensions,
|
|
134
|
+
roles: spec.roles,
|
|
135
|
+
transform: { prePasses: [schemaNameLiteralPass], assumeSchemasExist }
|
|
136
|
+
});
|
|
137
|
+
const transpiled = transpileBundle(source, {
|
|
138
|
+
renameChange: main.renameChange,
|
|
139
|
+
transformScript: main.transformScript,
|
|
140
|
+
provenance
|
|
141
|
+
});
|
|
142
|
+
if (main.result.errors.length > 0) {
|
|
143
|
+
const detail = main.result.errors.map(e => `${e.file}: ${e.error}`).join('; ');
|
|
144
|
+
throw new Error(`Reuse transpile of "${spec.source.module}" for "${perTenantName}" failed: ${detail}`);
|
|
145
|
+
}
|
|
146
|
+
// The partition is in *source* change names; splitBundle works on the
|
|
147
|
+
// transpiled bundle, so map the per-tenant set through the same rename.
|
|
148
|
+
const perTenantTranspiled = [...partition.perTenant].map(main.renameChange);
|
|
149
|
+
// 4. Per-tenant schema bootstrap: the source's `CREATE SCHEMA` lands in the
|
|
150
|
+
// shared module (shared target), but each tenant also needs its own
|
|
151
|
+
// schema. Duplicate it — transpiled to the per-tenant target — into the
|
|
152
|
+
// per-tenant module, re-pointing per-tenant dependencies off the shared
|
|
153
|
+
// schema onto this local one.
|
|
154
|
+
const bootstrap = buildSchemaBootstraps(source, spec, main.renameChange, assumeSchemasExist);
|
|
155
|
+
const { shared, perTenant } = splitBundle(transpiled, {
|
|
156
|
+
perTenantChanges: perTenantTranspiled,
|
|
157
|
+
sharedName,
|
|
158
|
+
perTenantName,
|
|
159
|
+
perTenantBootstrap: bootstrap
|
|
160
|
+
});
|
|
161
|
+
for (const [label, bundle] of [
|
|
162
|
+
[sharedName, shared],
|
|
163
|
+
[perTenantName, perTenant]
|
|
164
|
+
]) {
|
|
165
|
+
const issues = verifyBundle(bundle);
|
|
166
|
+
if (issues.length > 0) {
|
|
167
|
+
throw new Error(`Reuse bundle "${label}" failed integrity verification: ${issues.map(i => i.kind).join(', ')}`);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
const sharedOutDir = options.sharedOutDir ?? mkdtempSync(join(tmpdir(), `pgpm-apply-${sharedName}-`));
|
|
171
|
+
const perTenantOutDir = options.perTenantOutDir ?? mkdtempSync(join(tmpdir(), `pgpm-apply-${perTenantName}-`));
|
|
172
|
+
materializeBundle(shared, sharedOutDir);
|
|
173
|
+
materializeBundle(perTenant, perTenantOutDir);
|
|
174
|
+
return {
|
|
175
|
+
sharedName,
|
|
176
|
+
perTenantName,
|
|
177
|
+
shared: { bundle: shared, outDir: sharedOutDir },
|
|
178
|
+
perTenant: { bundle: perTenant, outDir: perTenantOutDir }
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Build a per-tenant `CREATE SCHEMA` bootstrap for each source schema that both
|
|
183
|
+
* creates a schema and has per-tenant objects. The scripts are the source
|
|
184
|
+
* schema-creation change transpiled to the per-tenant target; `replacesShared`
|
|
185
|
+
* is the shared-schema change the per-tenant objects would otherwise depend on.
|
|
186
|
+
*/
|
|
187
|
+
function buildSchemaBootstraps(source, spec, renameSharedChange, assumeSchemasExist) {
|
|
188
|
+
const bootstraps = [];
|
|
189
|
+
for (const [fromSchema, perTenantTarget] of Object.entries(spec.schemas)) {
|
|
190
|
+
const sharedTarget = spec.reuse.sharedSchema[fromSchema];
|
|
191
|
+
if (!sharedTarget || sharedTarget === perTenantTarget)
|
|
192
|
+
continue;
|
|
193
|
+
const sourceSchemaChange = `schemas/${fromSchema}/schema`;
|
|
194
|
+
const schemaChange = source.changes.find(c => c.name === sourceSchemaChange);
|
|
195
|
+
if (!schemaChange)
|
|
196
|
+
continue; // source doesn't create this schema; nothing to duplicate
|
|
197
|
+
// Transpile just the schema-creation change to the per-tenant target.
|
|
198
|
+
const single = {
|
|
199
|
+
...source,
|
|
200
|
+
manifest: { ...source.manifest, changeCount: 1, deployOrder: [sourceSchemaChange] },
|
|
201
|
+
changes: [schemaChange]
|
|
202
|
+
};
|
|
203
|
+
const t = makeSchemaTranspiler({
|
|
204
|
+
schemaMap: { [fromSchema]: perTenantTarget },
|
|
205
|
+
transform: { prePasses: [schemaNameLiteralPass], assumeSchemasExist }
|
|
206
|
+
});
|
|
207
|
+
const perTenantSchema = transpileBundle(single, {
|
|
208
|
+
renameChange: t.renameChange,
|
|
209
|
+
transformScript: t.transformScript
|
|
210
|
+
}).changes[0];
|
|
211
|
+
bootstraps.push({
|
|
212
|
+
name: `schemas/${perTenantTarget}/schema`,
|
|
213
|
+
dependencies: [],
|
|
214
|
+
deploy: perTenantSchema.deploy?.sql ?? null,
|
|
215
|
+
revert: perTenantSchema.revert?.sql ?? null,
|
|
216
|
+
verify: perTenantSchema.verify?.sql ?? null,
|
|
217
|
+
replacesShared: renameSharedChange(sourceSchemaChange)
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
return bootstraps;
|
|
221
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { existsSync, readdirSync, readFileSync } from 'fs';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { getExtensionName, parseControlContent } from '@pgpmjs/ast/files';
|
|
4
|
+
import { BUNDLE_ARCHIVE_EXTENSION, bundleFromModule, readBundleArchiveFile, withExecutableSql, writeBundleArchiveFile } from '@pgpmjs/bundle';
|
|
5
|
+
import { cleanSql } from '../migrate/clean';
|
|
6
|
+
/**
|
|
7
|
+
* Artifact file name for a module's stored bundle: it lives beside the packaged
|
|
8
|
+
* `sql/<name>--<version>.sql` as `sql/<name>--<version>.bundle.tar.gz`.
|
|
9
|
+
*/
|
|
10
|
+
export function bundleArtifactFileName(name, version) {
|
|
11
|
+
return `${name}--${version}${BUNDLE_ARCHIVE_EXTENSION}`;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Locate a module's stored bundle artifact, if it has one.
|
|
15
|
+
*
|
|
16
|
+
* Prefers the version declared in the module's `.control` file, then falls back
|
|
17
|
+
* to any single `sql/*.bundle.tar.gz` present — so a module packaged at a
|
|
18
|
+
* different version still gets the fast path instead of silently degrading.
|
|
19
|
+
* Returns `null` when the module ships no artifact.
|
|
20
|
+
*/
|
|
21
|
+
export function resolveBundleArtifactPath(moduleDir) {
|
|
22
|
+
const sqlDir = join(moduleDir, 'sql');
|
|
23
|
+
if (!existsSync(sqlDir))
|
|
24
|
+
return null;
|
|
25
|
+
let name;
|
|
26
|
+
try {
|
|
27
|
+
name = getExtensionName(moduleDir);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
const controlPath = join(moduleDir, `${name}.control`);
|
|
33
|
+
if (existsSync(controlPath)) {
|
|
34
|
+
const { version } = parseControlContent(readFileSync(controlPath, 'utf-8'));
|
|
35
|
+
if (version) {
|
|
36
|
+
const versioned = join(sqlDir, bundleArtifactFileName(name, version));
|
|
37
|
+
if (existsSync(versioned))
|
|
38
|
+
return versioned;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const candidates = readdirSync(sqlDir).filter(file => file.startsWith(`${name}--`) && file.endsWith(BUNDLE_ARCHIVE_EXTENSION));
|
|
42
|
+
if (candidates.length !== 1)
|
|
43
|
+
return null;
|
|
44
|
+
return join(sqlDir, candidates[0]);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Build the executable form of a module's bundle: the content-addressed AST
|
|
48
|
+
* snapshot plus, per change, the deploy SQL with transaction / `CREATE
|
|
49
|
+
* EXTENSION` statements stripped (`cleanSql`).
|
|
50
|
+
*
|
|
51
|
+
* That pre-computation is the whole point of the artifact — the parse/deparse
|
|
52
|
+
* work moves from every deploy to the one-time package step, while the digests
|
|
53
|
+
* still cover the exact bytes that will be executed.
|
|
54
|
+
*/
|
|
55
|
+
export async function buildExecutableBundle(moduleDir, options) {
|
|
56
|
+
const bundle = bundleFromModule(moduleDir, options);
|
|
57
|
+
const execSql = {};
|
|
58
|
+
for (const change of bundle.changes) {
|
|
59
|
+
if (!change.deploy)
|
|
60
|
+
continue;
|
|
61
|
+
execSql[change.name] = await cleanSql(change.deploy.sql, false, '$EOFCODE$');
|
|
62
|
+
}
|
|
63
|
+
return withExecutableSql(bundle, execSql);
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Emit a module's stored bundle artifact into `sql/`, returning its path.
|
|
67
|
+
*/
|
|
68
|
+
export async function writeBundleArtifact(moduleDir, version, options) {
|
|
69
|
+
const bundle = await buildExecutableBundle(moduleDir, options);
|
|
70
|
+
const outPath = join(moduleDir, 'sql', bundleArtifactFileName(bundle.manifest.name, version));
|
|
71
|
+
writeBundleArchiveFile(bundle, outPath);
|
|
72
|
+
return outPath;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Read a module's stored bundle artifact, or `null` when it has none or the
|
|
76
|
+
* archive cannot be read. Never throws — a broken artifact must degrade to the
|
|
77
|
+
* normal packaging path, not fail the deploy.
|
|
78
|
+
*/
|
|
79
|
+
export function readBundleArtifact(moduleDir) {
|
|
80
|
+
const artifactPath = resolveBundleArtifactPath(moduleDir);
|
|
81
|
+
if (!artifactPath)
|
|
82
|
+
return null;
|
|
83
|
+
try {
|
|
84
|
+
return readBundleArchiveFile(artifactPath);
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import { Logger } from '@pgpmjs/logger';
|
|
2
|
+
import { verifyBundle } from '@pgpmjs/bundle';
|
|
3
|
+
import { getPgPool } from 'pg-cache';
|
|
4
|
+
import { PgpmMigrate } from '../migrate/client';
|
|
5
|
+
import { buildExecutableBundle, readBundleArtifact } from './artifact';
|
|
6
|
+
const log = new Logger('deploy-fast');
|
|
7
|
+
/** Thrown when the ledger already holds a change with different content. */
|
|
8
|
+
export class BundledDeployConflictError extends Error {
|
|
9
|
+
packageName;
|
|
10
|
+
changeName;
|
|
11
|
+
constructor(packageName, changeName) {
|
|
12
|
+
super(`Change ${changeName} already deployed in package ${packageName} with different content`);
|
|
13
|
+
this.packageName = packageName;
|
|
14
|
+
this.changeName = changeName;
|
|
15
|
+
this.name = 'BundledDeployConflictError';
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* The ledger hash for a change: the sha256 of the deploy script bytes.
|
|
20
|
+
*
|
|
21
|
+
* This is deliberately the same value `PgpmMigrate.calculateScriptHash` derives
|
|
22
|
+
* from `deploy/<change>.sql` under the default `content` hash method, so a
|
|
23
|
+
* change hashed at package time and the hash recorded/compared at deploy time
|
|
24
|
+
* agree — the one-shot and per-change paths are interchangeable and idempotent
|
|
25
|
+
* against each other.
|
|
26
|
+
*/
|
|
27
|
+
export function ledgerHashForChange(change) {
|
|
28
|
+
return change.deploy?.digest ?? null;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Load a module's executable bundle: its stored artifact when one verifies,
|
|
32
|
+
* otherwise built from `deploy/` on the fly.
|
|
33
|
+
*
|
|
34
|
+
* The artifact is purely an optimization — it removes the read+parse work, not
|
|
35
|
+
* a capability. Whatever the source, the resulting bundle is byte-identical in
|
|
36
|
+
* what it executes and in the hashes it records, so a module without an
|
|
37
|
+
* artifact deploys exactly the same way, just slower.
|
|
38
|
+
*/
|
|
39
|
+
async function loadExecutableBundle(moduleDir) {
|
|
40
|
+
const stored = readBundleArtifact(moduleDir);
|
|
41
|
+
if (stored) {
|
|
42
|
+
const issues = verifyBundle(stored);
|
|
43
|
+
if (issues.length === 0) {
|
|
44
|
+
return { bundle: stored, source: 'artifact' };
|
|
45
|
+
}
|
|
46
|
+
log.warn(`⚠️ Bundle artifact for ${stored.manifest.name} failed verification ` +
|
|
47
|
+
`(${issues.map(i => i.kind).join(', ')}); rebuilding from deploy/.`);
|
|
48
|
+
}
|
|
49
|
+
return { bundle: await buildExecutableBundle(moduleDir), source: 'packaged' };
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Deploy a module in one shot **and** record the migration ledger — the `fast`
|
|
53
|
+
* deploy strategy.
|
|
54
|
+
*
|
|
55
|
+
* Pending changes' deploy SQL is concatenated and executed as a single
|
|
56
|
+
* statement (the speed of the old fast path), then one `pgpm_migrate.changes`
|
|
57
|
+
* row per change is bulk-inserted with the deploy script's own sha256 as
|
|
58
|
+
* `script_hash` — one round-trip, instead of an `isDeployed` + `readScript` +
|
|
59
|
+
* `CALL pgpm_migrate.deploy` per change. Because the ledger is written, the
|
|
60
|
+
* path is idempotent and resumable: re-deploys skip changes whose stored hash
|
|
61
|
+
* matches, and a same-name/different-content change raises
|
|
62
|
+
* {@link BundledDeployConflictError} exactly as the per-change path does.
|
|
63
|
+
*
|
|
64
|
+
* Returns a {@link BundledDeploySkipReason} (rather than throwing) when it
|
|
65
|
+
* cannot honour the requested semantics, so the caller falls back to the
|
|
66
|
+
* per-change migration path. Genuine SQL failures still throw.
|
|
67
|
+
*/
|
|
68
|
+
export async function deployModuleFast(moduleDir, options) {
|
|
69
|
+
if ((options.hashMethod ?? 'content') !== 'content') {
|
|
70
|
+
return 'unsupported-hash-method';
|
|
71
|
+
}
|
|
72
|
+
const { bundle, source } = await loadExecutableBundle(moduleDir);
|
|
73
|
+
const executable = bundle.changes.filter(change => change.deploy);
|
|
74
|
+
if (executable.length === 0 || executable.some(change => !change.exec)) {
|
|
75
|
+
return 'no-executable-sql';
|
|
76
|
+
}
|
|
77
|
+
const packageName = bundle.manifest.name;
|
|
78
|
+
const config = options.database
|
|
79
|
+
? { ...options.config, database: options.database }
|
|
80
|
+
: options.config;
|
|
81
|
+
// Ensures the pgpm_migrate schema/procedures exist, exactly as the
|
|
82
|
+
// per-change deploy path does.
|
|
83
|
+
await new PgpmMigrate(config, { hashMethod: 'content' }).initialize();
|
|
84
|
+
const pool = getPgPool(config);
|
|
85
|
+
const existing = await pool.query('SELECT change_name, script_hash FROM pgpm_migrate.changes WHERE package = $1', [packageName]);
|
|
86
|
+
const deployedHashes = new Map(existing.rows.map(row => [row.change_name, row.script_hash]));
|
|
87
|
+
const pending = [];
|
|
88
|
+
const skipped = [];
|
|
89
|
+
for (const change of executable) {
|
|
90
|
+
const hash = ledgerHashForChange(change);
|
|
91
|
+
const recorded = deployedHashes.get(change.name);
|
|
92
|
+
if (recorded === undefined) {
|
|
93
|
+
pending.push(change);
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (recorded !== hash) {
|
|
97
|
+
throw new BundledDeployConflictError(packageName, change.name);
|
|
98
|
+
}
|
|
99
|
+
skipped.push(change.name);
|
|
100
|
+
}
|
|
101
|
+
if (pending.length === 0) {
|
|
102
|
+
log.info(`⚡ ${packageName}: all ${skipped.length} change(s) already deployed.`);
|
|
103
|
+
return { name: packageName, source, deployed: [], skipped };
|
|
104
|
+
}
|
|
105
|
+
const changeNames = pending.map(change => change.name);
|
|
106
|
+
const hashes = pending.map(change => ledgerHashForChange(change));
|
|
107
|
+
const depChanges = [];
|
|
108
|
+
const depRequires = [];
|
|
109
|
+
for (const change of pending) {
|
|
110
|
+
for (const dep of new Set(change.dependencies)) {
|
|
111
|
+
depChanges.push(change.name);
|
|
112
|
+
depRequires.push(dep.includes(':') ? dep : `${packageName}:${dep}`);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const deploySql = pending.map(change => change.exec.sql).join('\n');
|
|
116
|
+
const client = await pool.connect();
|
|
117
|
+
const useTransaction = options.useTransaction ?? true;
|
|
118
|
+
try {
|
|
119
|
+
if (useTransaction)
|
|
120
|
+
await client.query('BEGIN');
|
|
121
|
+
if (!options.logOnly) {
|
|
122
|
+
await client.query(deploySql);
|
|
123
|
+
}
|
|
124
|
+
await client.query('INSERT INTO pgpm_migrate.packages (package) VALUES ($1) ON CONFLICT (package) DO NOTHING', [packageName]);
|
|
125
|
+
await client.query(`WITH input AS (
|
|
126
|
+
SELECT change_name, script_hash
|
|
127
|
+
FROM unnest($2::text[], $3::text[]) AS t(change_name, script_hash)
|
|
128
|
+
), inserted AS (
|
|
129
|
+
INSERT INTO pgpm_migrate.changes (change_id, change_name, package, script_hash)
|
|
130
|
+
SELECT encode(sha256(($1 || change_name || script_hash)::bytea), 'hex'),
|
|
131
|
+
change_name, $1, script_hash
|
|
132
|
+
FROM input
|
|
133
|
+
ON CONFLICT (package, change_name) DO NOTHING
|
|
134
|
+
RETURNING change_id, change_name
|
|
135
|
+
), deps AS (
|
|
136
|
+
INSERT INTO pgpm_migrate.dependencies (change_id, requires)
|
|
137
|
+
SELECT inserted.change_id, d.requires
|
|
138
|
+
FROM unnest($4::text[], $5::text[]) AS d(change_name, requires)
|
|
139
|
+
JOIN inserted ON inserted.change_name = d.change_name
|
|
140
|
+
ON CONFLICT DO NOTHING
|
|
141
|
+
RETURNING change_id
|
|
142
|
+
)
|
|
143
|
+
INSERT INTO pgpm_migrate.events (event_type, change_name, package)
|
|
144
|
+
SELECT 'deploy', change_name, $1 FROM inserted`, [packageName, changeNames, hashes, depChanges, depRequires]);
|
|
145
|
+
if (useTransaction)
|
|
146
|
+
await client.query('COMMIT');
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
if (useTransaction) {
|
|
150
|
+
try {
|
|
151
|
+
await client.query('ROLLBACK');
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
// the connection is already unusable; surface the original error
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
159
|
+
finally {
|
|
160
|
+
client.release();
|
|
161
|
+
}
|
|
162
|
+
log.success(`⚡ ${packageName}: ${changeNames.length} change(s) ${options.logOnly ? 'logged' : 'deployed'} ` +
|
|
163
|
+
`from ${source === 'artifact' ? 'bundle artifact' : 'packaged deploy/'} ` +
|
|
164
|
+
`(${skipped.length} already deployed).`);
|
|
165
|
+
return { name: packageName, source, deployed: changeNames, skipped };
|
|
166
|
+
}
|
|
167
|
+
/** True when the value is a real deploy result rather than a skip reason. */
|
|
168
|
+
export function isBundledDeployResult(value) {
|
|
169
|
+
return typeof value !== 'string';
|
|
170
|
+
}
|
package/esm/bundle/index.js
CHANGED
|
@@ -15,3 +15,7 @@ export * from '@pgpmjs/bundle';
|
|
|
15
15
|
export * from './apply';
|
|
16
16
|
// applyEnvelope: schema apply + data/fixtures part replay for a BundleEnvelope.
|
|
17
17
|
export * from './apply-envelope';
|
|
18
|
+
// The stored `sql/<name>--<version>.bundle.tar.gz` artifact: emit + resolve.
|
|
19
|
+
export * from './artifact';
|
|
20
|
+
// The bundle-backed deploy fast path (one-shot execute + ledger).
|
|
21
|
+
export * from './deploy-bundled';
|
package/esm/core/class/pgpm.js
CHANGED
|
@@ -11,6 +11,7 @@ import { getPgPool } from 'pg-cache';
|
|
|
11
11
|
import yanse from 'yanse';
|
|
12
12
|
import { resolveEffectiveModulePath } from '../../apply/materialize';
|
|
13
13
|
import { hasApplySpec, readApplySpec } from '../../apply/apply-spec';
|
|
14
|
+
import { isReuseSpec, resolveSharedModuleName } from '../../apply/reuse';
|
|
14
15
|
import { APPLY_SPEC_FILE } from '../../apply/types';
|
|
15
16
|
import { getAvailableExtensions } from '../../extensions/extensions';
|
|
16
17
|
import { generatePlan, writePlan, writePlanFile } from '@pgpmjs/ast/files';
|
|
@@ -21,7 +22,7 @@ import { parsePlanFile } from '@pgpmjs/ast/files/plan/parser';
|
|
|
21
22
|
import { isValidChangeName, isValidTagName, parseReference } from '@pgpmjs/ast/files/plan/validators';
|
|
22
23
|
import { PgpmMigrate } from '../../migrate/client';
|
|
23
24
|
import { getExtensionsAndModules, getExtensionsAndModulesChanges, latestChange, latestChangeAndVersion } from '../../modules/modules';
|
|
24
|
-
import {
|
|
25
|
+
import { deployModuleFast, isBundledDeployResult } from '../../bundle/deploy-bundled';
|
|
25
26
|
import { syncModuleVersions } from '../../packaging/sync-versions';
|
|
26
27
|
import { resolveDependencies, resolveExtensionDependencies } from '../../resolution/deps';
|
|
27
28
|
import { movePath } from '../../utils/fs';
|
|
@@ -274,13 +275,30 @@ export class PgpmPackage {
|
|
|
274
275
|
if (moduleMap[name])
|
|
275
276
|
continue;
|
|
276
277
|
const source = moduleMap[spec.source.module];
|
|
277
|
-
|
|
278
|
-
path
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
278
|
+
const relPath = path
|
|
279
|
+
.dirname(path.relative(this.workspacePath, file))
|
|
280
|
+
.replace(/\\/g, '/');
|
|
281
|
+
const baseRequires = spec.requires ?? source?.requires ?? [];
|
|
282
|
+
const version = spec.version ?? source?.version ?? '0.0.1';
|
|
283
|
+
if (isReuseSpec(spec)) {
|
|
284
|
+
// A reuse proxy expands into two modules that both materialize from
|
|
285
|
+
// this one directory: a shared module (deployed once, reused across
|
|
286
|
+
// tenants) and the per-tenant module that `requires` it. Multiple
|
|
287
|
+
// tenant proxies over the same source/shared/seed config resolve to
|
|
288
|
+
// the same deterministic shared name, so it is added at most once.
|
|
289
|
+
const sharedName = resolveSharedModuleName(spec);
|
|
290
|
+
if (!moduleMap[sharedName]) {
|
|
291
|
+
moduleMap[sharedName] = { path: relPath, requires: baseRequires, version };
|
|
292
|
+
}
|
|
293
|
+
moduleMap[name] = {
|
|
294
|
+
path: relPath,
|
|
295
|
+
requires: [...baseRequires, sharedName],
|
|
296
|
+
version
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
else {
|
|
300
|
+
moduleMap[name] = { path: relPath, requires: baseRequires, version };
|
|
301
|
+
}
|
|
284
302
|
}
|
|
285
303
|
return moduleMap;
|
|
286
304
|
}
|
|
@@ -1295,12 +1313,6 @@ ${dependencies.length > 0 ? dependencies.map(dep => `-- requires: ${dep}`).join(
|
|
|
1295
1313
|
const log = new Logger('deploy');
|
|
1296
1314
|
const { name, toChange } = this.parsePackageTarget(target);
|
|
1297
1315
|
if (recursive) {
|
|
1298
|
-
// Cache for fast deployment
|
|
1299
|
-
const deployFastCache = {};
|
|
1300
|
-
const getCacheKey = (pg, name, database) => {
|
|
1301
|
-
const { host, port, user } = pg ?? {};
|
|
1302
|
-
return `${host}:${port}:${user}:${database}:${name}`;
|
|
1303
|
-
};
|
|
1304
1316
|
const modules = this.getModuleMap();
|
|
1305
1317
|
let extensions;
|
|
1306
1318
|
if (name === null) {
|
|
@@ -1323,49 +1335,22 @@ ${dependencies.length > 0 ? dependencies.map(dep => `-- requires: ${dep}`).join(
|
|
|
1323
1335
|
else {
|
|
1324
1336
|
const modulePath = await resolveEffectiveModulePath(extension, resolve(this.workspacePath, modules[extension].path), modules, this.workspacePath);
|
|
1325
1337
|
log.info(`📂 Deploying local module: ${extension}`);
|
|
1326
|
-
if (opts.deployment.fast) {
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1338
|
+
if (opts.deployment.fast || opts.deployment.bundled) {
|
|
1339
|
+
// Fast strategy: one-shot execution plus a bulk-written
|
|
1340
|
+
// pgpm_migrate ledger, from the module's verified bundle artifact
|
|
1341
|
+
// when it has one and from `deploy/` when it does not.
|
|
1342
|
+
const outcome = await deployModuleFast(modulePath, {
|
|
1343
|
+
config: opts.pg,
|
|
1344
|
+
logOnly: opts.deployment.logOnly,
|
|
1345
|
+
useTransaction: opts.deployment.useTx,
|
|
1346
|
+
hashMethod: opts.deployment.hashMethod
|
|
1347
|
+
});
|
|
1348
|
+
if (isBundledDeployResult(outcome)) {
|
|
1331
1349
|
continue;
|
|
1332
1350
|
}
|
|
1333
|
-
|
|
1334
|
-
try {
|
|
1335
|
-
pkg = await packageModule(modulePath, {
|
|
1336
|
-
usePlan: opts.deployment.usePlan,
|
|
1337
|
-
extension: false
|
|
1338
|
-
});
|
|
1339
|
-
}
|
|
1340
|
-
catch (err) {
|
|
1341
|
-
const errorLines = [];
|
|
1342
|
-
errorLines.push(`❌ Failed to package module "${extension}" at path: ${modulePath}`);
|
|
1343
|
-
errorLines.push(` Module Path: ${modulePath}`);
|
|
1344
|
-
errorLines.push(` Workspace Path: ${this.workspacePath}`);
|
|
1345
|
-
errorLines.push(` Error Code: ${err.code || 'N/A'}`);
|
|
1346
|
-
errorLines.push(` Error Message: ${err.message || 'Unknown error'}`);
|
|
1347
|
-
if (err.code === 'ENOENT') {
|
|
1348
|
-
errorLines.push('💡 Hint: File or directory not found. Check if the module path is correct.');
|
|
1349
|
-
}
|
|
1350
|
-
else if (err.code === 'EACCES') {
|
|
1351
|
-
errorLines.push('💡 Hint: Permission denied. Check file permissions.');
|
|
1352
|
-
}
|
|
1353
|
-
else if (err.message && err.message.includes('pgpm.plan')) {
|
|
1354
|
-
errorLines.push('💡 Hint: pgpm.plan file issue. Check if the plan file exists and is valid.');
|
|
1355
|
-
}
|
|
1356
|
-
log.error(errorLines.join('\n'));
|
|
1357
|
-
console.error(err);
|
|
1358
|
-
throw errors.DEPLOYMENT_FAILED({
|
|
1359
|
-
type: 'Deployment',
|
|
1360
|
-
module: extension
|
|
1361
|
-
});
|
|
1362
|
-
}
|
|
1363
|
-
await pgPool.query(pkg.sql);
|
|
1364
|
-
if (opts.deployment.cache) {
|
|
1365
|
-
deployFastCache[cacheKey] = pkg;
|
|
1366
|
-
}
|
|
1351
|
+
log.info(`↩️ Fast deploy unavailable for ${extension} (${outcome}); using per-change path.`);
|
|
1367
1352
|
}
|
|
1368
|
-
|
|
1353
|
+
{
|
|
1369
1354
|
try {
|
|
1370
1355
|
const client = new PgpmMigrate(opts.pg);
|
|
1371
1356
|
// Only apply toChange to the target module, not its dependencies
|
package/esm/packaging/package.js
CHANGED
|
@@ -4,6 +4,7 @@ import { relative } from 'path';
|
|
|
4
4
|
import { deparse } from 'pgsql-deparser';
|
|
5
5
|
import { parse } from 'pgsql-parser';
|
|
6
6
|
import { getExtensionName } from '@pgpmjs/ast/files';
|
|
7
|
+
import { writeBundleArtifact } from '../bundle/artifact';
|
|
7
8
|
import { resolve, resolveWithPlan } from '../resolution/resolve';
|
|
8
9
|
import { transformProps } from './transform';
|
|
9
10
|
const log = new Logger('package');
|
|
@@ -78,7 +79,7 @@ export const packageModule = async (packageDir, { usePlan = true, extension = tr
|
|
|
78
79
|
throw e;
|
|
79
80
|
}
|
|
80
81
|
};
|
|
81
|
-
export const writePackage = async ({ version, extension = true, usePlan = true, packageDir, outputDiff = false, }) => {
|
|
82
|
+
export const writePackage = async ({ version, extension = true, usePlan = true, packageDir, outputDiff = false, bundle = true, }) => {
|
|
82
83
|
const pkgPath = `${packageDir}/package.json`;
|
|
83
84
|
const pkg = require(pkgPath);
|
|
84
85
|
const extname = await getExtensionName(packageDir);
|
|
@@ -115,4 +116,10 @@ export const writePackage = async ({ version, extension = true, usePlan = true,
|
|
|
115
116
|
const writePath = `${outPath}/${sqlFileName}`;
|
|
116
117
|
writeFileSync(writePath, sql);
|
|
117
118
|
log.success(`${relative(packageDir, writePath)} written`);
|
|
119
|
+
if (bundle) {
|
|
120
|
+
const bundlePath = await writeBundleArtifact(packageDir, version, {
|
|
121
|
+
createdWith: '@pgpmjs/core'
|
|
122
|
+
});
|
|
123
|
+
log.success(`${relative(packageDir, bundlePath)} written`);
|
|
124
|
+
}
|
|
118
125
|
};
|