@decocms/blocks-cli 8.0.0 → 8.1.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -3
- package/scripts/analyze-traces.mjs +1 -0
- package/scripts/cdn-rules.test.ts +105 -0
- package/scripts/cdn-rules.ts +192 -0
- package/scripts/deco-migrate-cli.ts +6 -7
- package/scripts/fast-deploy-kv.test.ts +172 -0
- package/scripts/generate-blocks.test.ts +11 -0
- package/scripts/generate-blocks.ts +0 -10
- package/scripts/generate-invoke.test.ts +2 -2
- package/scripts/generate-invoke.ts +9 -21
- package/scripts/generate-loaders.test.ts +2 -2
- package/scripts/generate-loaders.ts +0 -10
- package/scripts/generate-schema-loader-aliases.test.ts +186 -0
- package/scripts/generate-schema-workspaces.test.ts +180 -0
- package/scripts/generate-schema.test.ts +68 -29
- package/scripts/generate-schema.ts +101 -89
- package/scripts/generate-sections.test.ts +104 -35
- package/scripts/generate-sections.ts +57 -11
- package/scripts/generate-workspaces.test.ts +179 -0
- package/scripts/generate.test.ts +1 -2
- package/scripts/generate.ts +43 -45
- package/scripts/lib/blocks-dedupe.test.ts +1 -1
- package/scripts/lib/cf-kv-rest.ts +49 -1
- package/scripts/lib/installedPackages.ts +42 -0
- package/scripts/lib/invokeSource.ts +18 -0
- package/scripts/lib/kv-snapshot.ts +44 -0
- package/scripts/lib/read-decofile.ts +13 -2
- package/scripts/lib/upgrade-6-to-7.ts +1 -1
- package/scripts/lib/wrangler-config.test.ts +49 -0
- package/scripts/lib/wrangler-config.ts +31 -0
- package/scripts/migrate/analyzers/htmx-analyze.test.ts +6 -6
- package/scripts/migrate/analyzers/tailwind-config.ts +95 -1
- package/scripts/migrate/config.ts +2 -2
- package/scripts/migrate/delete-sets.ts +75 -0
- package/scripts/migrate/fast-deploy-scaffold.test.ts +130 -0
- package/scripts/migrate/icon-casing.test.ts +63 -0
- package/scripts/migrate/phase-analyze.test.ts +66 -4
- package/scripts/migrate/phase-analyze.ts +79 -60
- package/scripts/migrate/phase-cleanup.test.ts +3 -1
- package/scripts/migrate/phase-cleanup.ts +1 -1
- package/scripts/migrate/phase-compile.test.ts +3 -1
- package/scripts/migrate/phase-report.test.ts +2 -0
- package/scripts/migrate/phase-report.ts +3 -4
- package/scripts/migrate/phase-scaffold.ts +150 -45
- package/scripts/migrate/phase-transform.ts +107 -2
- package/scripts/migrate/phase-verify-legacy-specifiers.test.ts +13 -0
- package/scripts/migrate/phase-verify.test.ts +3 -1
- package/scripts/migrate/phase-verify.ts +87 -33
- package/scripts/migrate/post-cleanup/rules.ts +29 -11
- package/scripts/migrate/post-cleanup/shim-classify.test.ts +1 -1
- package/scripts/migrate/post-cleanup/shim-classify.ts +2 -2
- package/scripts/migrate/source-layout.test.ts +43 -0
- package/scripts/migrate/source-layout.ts +32 -5
- package/scripts/migrate/templates/app-css.test.ts +24 -0
- package/scripts/migrate/templates/app-css.ts +27 -0
- package/scripts/migrate/templates/ci-workflows.test.ts +179 -0
- package/scripts/migrate/templates/ci-yml.ts +182 -0
- package/scripts/migrate/templates/cursor-rules.test.ts +3 -3
- package/scripts/migrate/templates/hooks.test.ts +3 -1
- package/scripts/migrate/templates/lib-utils.ts +3 -3
- package/scripts/migrate/templates/main-push-guard-yml.ts +45 -0
- package/scripts/migrate/templates/package-json.ts +10 -0
- package/scripts/migrate/templates/parity-yml.ts +136 -0
- package/scripts/migrate/templates/perf-yml.ts +449 -0
- package/scripts/migrate/templates/playwright-yml.ts +127 -0
- package/scripts/migrate/templates/react-doctor-yml.ts +46 -0
- package/scripts/migrate/templates/routes.test.ts +31 -0
- package/scripts/migrate/templates/routes.ts +19 -9
- package/scripts/migrate/templates/section-loaders.ts +3 -3
- package/scripts/migrate/templates/server-entry-vtex-invoke.test.ts +19 -0
- package/scripts/migrate/templates/server-entry.ts +19 -14
- package/scripts/migrate/templates/setup.ts +11 -3
- package/scripts/migrate/templates/sync-blocks-bot-yml.ts +201 -0
- package/scripts/migrate/templates/ui-components.ts +3 -0
- package/scripts/migrate/transforms/ctx-compat.test.ts +37 -0
- package/scripts/migrate/transforms/ctx-compat.ts +93 -1
- package/scripts/migrate/transforms/htmx-on-events.test.ts +1 -1
- package/scripts/migrate/transforms/htmx-on-events.ts +2 -2
- package/scripts/migrate/transforms/jsx-class-type.test.ts +28 -0
- package/scripts/migrate/transforms/jsx.ts +10 -0
- package/scripts/migrate/transforms/picture.test.ts +56 -0
- package/scripts/migrate/transforms/picture.ts +95 -0
- package/scripts/migrate/transforms/tailwind-renames.test.ts +29 -0
- package/scripts/migrate/transforms/tailwind-renames.ts +34 -0
- package/scripts/migrate/transforms/timer-types.test.ts +30 -0
- package/scripts/migrate/transforms/timer-types.ts +48 -0
- package/scripts/migrate/transforms/use-script-handlers.test.ts +37 -0
- package/scripts/migrate/transforms/use-script-handlers.ts +36 -0
- package/scripts/migrate/types.ts +13 -1
- package/scripts/migrate-blocks-to-kv.ts +16 -2
- package/scripts/migrate-to-cf-observability.test.ts +5 -5
- package/scripts/migrate-to-cf-observability.ts +1 -1
- package/scripts/migrate.ts +244 -6
- package/scripts/reconcile.test.ts +162 -0
- package/scripts/reconcile.ts +303 -0
- package/scripts/sync-blocks-bot.test.ts +276 -0
- package/scripts/sync-blocks-bot.ts +567 -0
- package/scripts/sync-blocks-to-kv.ts +20 -2
- package/scripts/upgrade-6-to-7.ts +0 -0
- package/scripts/lib/legacyArtifact.ts +0 -20
|
@@ -0,0 +1,567 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* @decocms/blocks-cli — pull the production decofile into `.deco/blocks/`
|
|
4
|
+
*
|
|
5
|
+
* Secure replacement for the legacy push-based content sync, where a workflow
|
|
6
|
+
* in the *legacy* Fresh repo held a cross-repo PAT and pushed straight into the
|
|
7
|
+
* migrated repo's `main` (`rsync --delete` + `git push`). That token is a code
|
|
8
|
+
* -write door into the new repo, not a content channel.
|
|
9
|
+
*
|
|
10
|
+
* This inverts the direction: the migrated repo *pulls* the decofile from the
|
|
11
|
+
* live site (`GET <origin>/.decofile`, public and unauthenticated) on a daily
|
|
12
|
+
* cron, materialises one file per block, and opens a PR. No cross-repo token,
|
|
13
|
+
* no write permission handed to anyone, and the content passes a build gate
|
|
14
|
+
* before reaching `main`. See `docs/sync-blocks-bot.md`.
|
|
15
|
+
*
|
|
16
|
+
* Three filters decide what may be overwritten:
|
|
17
|
+
* 1. `--deny <globs>` — deny by block key (default: the `Site` block).
|
|
18
|
+
* 2. encrypted-secret shape — any block carrying a `{name, encrypted}` secret
|
|
19
|
+
* ref anywhere in its tree is left alone. That is what protects the
|
|
20
|
+
* credentials the migration moves onto the new site's own app block
|
|
21
|
+
* (e.g. `deco-vtex`), which do not exist in that layout upstream.
|
|
22
|
+
* Opt out with `--allow-secret-blocks`.
|
|
23
|
+
* 3. `--fail-on-plaintext-secret` — aborts if an accepted block carries what
|
|
24
|
+
* looks like a *plaintext* credential, so a leak upstream is never
|
|
25
|
+
* committed into git.
|
|
26
|
+
*
|
|
27
|
+
* Usage (from a site root):
|
|
28
|
+
* tsx sync-blocks-bot.ts --origin https://www.minhaloja.com.br --prune
|
|
29
|
+
* tsx sync-blocks-bot.ts --url https://www.minhaloja.com.br/.decofile --dry-run --json
|
|
30
|
+
*
|
|
31
|
+
* Exit codes:
|
|
32
|
+
* 0 — done (with or without changes)
|
|
33
|
+
* 1 — `--fail-on-plaintext-secret` and at least one plaintext finding
|
|
34
|
+
* 2 — usage / network / payload validation error (nothing was written)
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
import * as fs from "node:fs";
|
|
38
|
+
import * as path from "node:path";
|
|
39
|
+
import { fileURLToPath } from "node:url";
|
|
40
|
+
import { decodeBlockNameWithPasses } from "./lib/blocks-dedupe";
|
|
41
|
+
|
|
42
|
+
/** Block keys never overwritten by a sync unless `--deny` is overridden. */
|
|
43
|
+
export const DEFAULT_DENY = ["Site", "site"];
|
|
44
|
+
|
|
45
|
+
const DEFAULT_MAX_BYTES = 64 * 1024 * 1024;
|
|
46
|
+
const DEFAULT_TIMEOUT_MS = 60_000;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Property names whose *string* value would be a credential in the clear.
|
|
50
|
+
* A bare `key` is deliberately NOT here: CMS content is full of `key` props
|
|
51
|
+
* that are nothing of the sort (`selectedFacets[].key` on every VTEX PLP
|
|
52
|
+
* loader — 331 false positives on a real site), and a gate that cries wolf is a
|
|
53
|
+
* gate nobody keeps on.
|
|
54
|
+
* `key` only counts when qualified (`apiKey`, `appKey`, `privateKey`, …).
|
|
55
|
+
*/
|
|
56
|
+
const SECRET_PROP_RE =
|
|
57
|
+
/(?:^|_)(?:(?:api|app|access|private|public|client|secret|auth)_?keys?|tokens?|secrets?|passwords?|passwd|pwd)$/;
|
|
58
|
+
|
|
59
|
+
/** `appToken` -> `app_token`, so one snake_case regex covers both styles. */
|
|
60
|
+
function normalizeProp(prop: string): string {
|
|
61
|
+
return prop
|
|
62
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1_$2")
|
|
63
|
+
.replace(/-/g, "_")
|
|
64
|
+
.toLowerCase();
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface PullOptions {
|
|
68
|
+
/** Directory that holds one JSON file per block (usually `.deco/blocks`). */
|
|
69
|
+
out: string;
|
|
70
|
+
/** Glob patterns (`*` wildcard) matched against the block key. */
|
|
71
|
+
deny?: string[];
|
|
72
|
+
/** Overwrite blocks that carry an encrypted secret ref (default: false). */
|
|
73
|
+
allowSecretBlocks?: boolean;
|
|
74
|
+
/** Delete local blocks that no longer exist upstream (default: false). */
|
|
75
|
+
prune?: boolean;
|
|
76
|
+
/** Compute the report without touching the filesystem. */
|
|
77
|
+
dryRun?: boolean;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface PullReport {
|
|
81
|
+
added: string[];
|
|
82
|
+
updated: string[];
|
|
83
|
+
unchanged: number;
|
|
84
|
+
removed: string[];
|
|
85
|
+
/** Keys skipped by the deny glob. */
|
|
86
|
+
denied: string[];
|
|
87
|
+
/** Keys skipped because the local/remote block carries an encrypted secret. */
|
|
88
|
+
protectedSecretBlocks: string[];
|
|
89
|
+
/** Keys skipped because the payload value was not a JSON object. */
|
|
90
|
+
skipped: string[];
|
|
91
|
+
/** `<key>.<prop path>` of every plaintext credential found in a written block. */
|
|
92
|
+
plaintextSecrets: string[];
|
|
93
|
+
/** Blocks present in the remote payload. */
|
|
94
|
+
remoteBlocks: number;
|
|
95
|
+
revision?: string;
|
|
96
|
+
bytes?: number;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export interface FetchResult {
|
|
100
|
+
blocks: Record<string, unknown>;
|
|
101
|
+
revision?: string;
|
|
102
|
+
bytes: number;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// ---------------------------------------------------------------- pure helpers
|
|
106
|
+
|
|
107
|
+
/** `*`-only glob match against a whole block key. */
|
|
108
|
+
export function matchesGlob(key: string, pattern: string): boolean {
|
|
109
|
+
const rx = new RegExp(
|
|
110
|
+
`^${pattern.replace(/[.*+?^${}()|[\]\\]/g, (c) => (c === "*" ? "[\\s\\S]*" : `\\${c}`))}$`,
|
|
111
|
+
);
|
|
112
|
+
return rx.test(key);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
|
116
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* True iff the block carries a Deco encrypted-secret ref anywhere in its tree.
|
|
121
|
+
* Prod serves those as `{"name": "MY_TOKEN", "encrypted": "3a714de1c8…"}`.
|
|
122
|
+
*/
|
|
123
|
+
export function hasEncryptedSecretRef(value: unknown): boolean {
|
|
124
|
+
if (Array.isArray(value)) return value.some(hasEncryptedSecretRef);
|
|
125
|
+
if (!isPlainObject(value)) return false;
|
|
126
|
+
if (typeof value.name === "string" && typeof value.encrypted === "string" && value.encrypted) {
|
|
127
|
+
return true;
|
|
128
|
+
}
|
|
129
|
+
return Object.values(value).some(hasEncryptedSecretRef);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Property paths whose name looks credential-shaped and whose value is a raw
|
|
134
|
+
* string — i.e. a secret in the clear rather than an `{name, encrypted}` ref.
|
|
135
|
+
* URLs and values with whitespace are excluded (endpoints, prose, templates).
|
|
136
|
+
*/
|
|
137
|
+
export function findPlaintextSecrets(value: unknown, prefix = ""): string[] {
|
|
138
|
+
const found: string[] = [];
|
|
139
|
+
const walk = (node: unknown, at: string): void => {
|
|
140
|
+
if (Array.isArray(node)) {
|
|
141
|
+
for (const [i, item] of node.entries()) walk(item, `${at}[${i}]`);
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
if (!isPlainObject(node)) return;
|
|
145
|
+
for (const [prop, child] of Object.entries(node)) {
|
|
146
|
+
const at2 = at ? `${at}.${prop}` : prop;
|
|
147
|
+
if (
|
|
148
|
+
typeof child === "string" &&
|
|
149
|
+
child.length >= 12 &&
|
|
150
|
+
SECRET_PROP_RE.test(normalizeProp(prop)) &&
|
|
151
|
+
!/\s/.test(child) &&
|
|
152
|
+
!/^https?:\/\//i.test(child) &&
|
|
153
|
+
!/^\{\{.*\}\}$/.test(child)
|
|
154
|
+
) {
|
|
155
|
+
found.push(at2);
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
walk(child, at2);
|
|
159
|
+
}
|
|
160
|
+
};
|
|
161
|
+
walk(value, prefix);
|
|
162
|
+
return found;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Fully URL-decode a block key or filename stem, for cross-scheme matching. */
|
|
166
|
+
function canonicalKey(keyOrFile: string): string {
|
|
167
|
+
return decodeBlockNameWithPasses(keyOrFile).name;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Serialised on-disk form of a block: pretty-printed, so a PR diff shows the
|
|
172
|
+
* sections/props that actually changed instead of one 3 MB line.
|
|
173
|
+
*/
|
|
174
|
+
function serialize(block: unknown): string {
|
|
175
|
+
return `${JSON.stringify(block, null, 2)}\n`;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Key-sorted JSON, for *comparison only*. Writers of `.deco/blocks` disagree on
|
|
180
|
+
* formatting (the Studio daemon and the old bot minify, a hand-made sync PR
|
|
181
|
+
* pretty-prints) and on key order, so a byte compare reports a diff on a block
|
|
182
|
+
* whose content is identical — 124 of 432 blocks on a real site's first run.
|
|
183
|
+
* Compare semantically, write only real content changes.
|
|
184
|
+
*/
|
|
185
|
+
function stableStringify(value: unknown): string {
|
|
186
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
187
|
+
if (!isPlainObject(value)) return JSON.stringify(value) ?? "null";
|
|
188
|
+
const entries = Object.keys(value)
|
|
189
|
+
.sort()
|
|
190
|
+
.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`);
|
|
191
|
+
return `{${entries.join(",")}}`;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// ------------------------------------------------------------------ fetch side
|
|
195
|
+
|
|
196
|
+
/** Download and validate the remote decofile. Throws on any bad payload. */
|
|
197
|
+
export async function fetchDecofile(
|
|
198
|
+
url: string,
|
|
199
|
+
opts: { maxBytes?: number; timeoutMs?: number } = {},
|
|
200
|
+
): Promise<FetchResult> {
|
|
201
|
+
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
202
|
+
const res = await fetch(url, {
|
|
203
|
+
redirect: "follow",
|
|
204
|
+
headers: { accept: "application/json" },
|
|
205
|
+
signal: AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
|
206
|
+
});
|
|
207
|
+
if (!res.ok) throw new Error(`${url} responded ${res.status} ${res.statusText}`);
|
|
208
|
+
|
|
209
|
+
const contentType = res.headers.get("content-type") ?? "";
|
|
210
|
+
if (!contentType.includes("json")) {
|
|
211
|
+
throw new Error(`${url} served "${contentType || "no content-type"}", expected JSON`);
|
|
212
|
+
}
|
|
213
|
+
const declared = Number(res.headers.get("content-length") ?? Number.NaN);
|
|
214
|
+
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
215
|
+
throw new Error(`${url} is ${declared} bytes, over the ${maxBytes} byte cap`);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const body = await res.text();
|
|
219
|
+
if (body.length > maxBytes) {
|
|
220
|
+
throw new Error(`${url} is ${body.length} bytes, over the ${maxBytes} byte cap`);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
let parsed: unknown;
|
|
224
|
+
try {
|
|
225
|
+
parsed = JSON.parse(body);
|
|
226
|
+
} catch (e) {
|
|
227
|
+
throw new Error(`${url} did not return valid JSON: ${(e as Error).message}`);
|
|
228
|
+
}
|
|
229
|
+
if (!isPlainObject(parsed)) {
|
|
230
|
+
throw new Error(
|
|
231
|
+
`${url} returned ${Array.isArray(parsed) ? "an array" : typeof parsed}, expected a decofile object`,
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
return { blocks: parsed, revision: res.headers.get("etag") ?? undefined, bytes: body.length };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// ------------------------------------------------------------------ write side
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Materialise `remote` into `opts.out`, one file per block.
|
|
242
|
+
*
|
|
243
|
+
* Filenames are `encodeURIComponent(key) + ".json"` — the single-decode scheme
|
|
244
|
+
* the runtime's `parseBlockId` expects. When a file for the same *logical* key
|
|
245
|
+
* already exists under a different encoding (the `deco-sync-bot` wrote
|
|
246
|
+
* double-encoded names), that existing file is overwritten in place instead of
|
|
247
|
+
* a second, colliding one being created.
|
|
248
|
+
*/
|
|
249
|
+
export function writeDecofileToDir(remote: Record<string, unknown>, opts: PullOptions): PullReport {
|
|
250
|
+
const out = path.resolve(opts.out);
|
|
251
|
+
const deny = opts.deny ?? DEFAULT_DENY;
|
|
252
|
+
const report: PullReport = {
|
|
253
|
+
added: [],
|
|
254
|
+
updated: [],
|
|
255
|
+
unchanged: 0,
|
|
256
|
+
removed: [],
|
|
257
|
+
denied: [],
|
|
258
|
+
protectedSecretBlocks: [],
|
|
259
|
+
skipped: [],
|
|
260
|
+
plaintextSecrets: [],
|
|
261
|
+
remoteBlocks: Object.keys(remote).length,
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
fs.mkdirSync(out, { recursive: true });
|
|
265
|
+
|
|
266
|
+
// Index what is already on disk by fully-decoded key, so we overwrite legacy
|
|
267
|
+
// double-encoded filenames instead of duplicating them. A key can map to
|
|
268
|
+
// several files (the `deco-sync-bot` wrote `pages-A%2520B.json` where the
|
|
269
|
+
// manual sync wrote `pages-A%20B.json`).
|
|
270
|
+
const existingByKey = new Map<string, string[]>();
|
|
271
|
+
for (const file of fs.readdirSync(out)) {
|
|
272
|
+
if (!file.endsWith(".json")) continue;
|
|
273
|
+
const key = canonicalKey(file);
|
|
274
|
+
const list = existingByKey.get(key);
|
|
275
|
+
if (list) list.push(file);
|
|
276
|
+
else existingByKey.set(key, [file]);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const seen = new Set<string>();
|
|
280
|
+
|
|
281
|
+
for (const [key, block] of Object.entries(remote)) {
|
|
282
|
+
const canonical = canonicalKey(key);
|
|
283
|
+
seen.add(canonical);
|
|
284
|
+
|
|
285
|
+
if (deny.some((p) => matchesGlob(key, p) || matchesGlob(canonical, p))) {
|
|
286
|
+
report.denied.push(key);
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
289
|
+
if (!isPlainObject(block)) {
|
|
290
|
+
report.skipped.push(key);
|
|
291
|
+
continue;
|
|
292
|
+
}
|
|
293
|
+
if (!opts.allowSecretBlocks && hasEncryptedSecretRef(block)) {
|
|
294
|
+
report.protectedSecretBlocks.push(key);
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// One existing file → write it in place (smallest diff, no rename churn).
|
|
299
|
+
// Several → converge on the canonical single-encoded name and drop the
|
|
300
|
+
// others: leaving a stale duplicate behind is not cosmetic, `pickWinner`
|
|
301
|
+
// in generate-blocks prefers the *more*-encoded filename, so the stale one
|
|
302
|
+
// would win the build.
|
|
303
|
+
const existing = existingByKey.get(canonical) ?? [];
|
|
304
|
+
const canonicalFile = `${encodeURIComponent(key)}.json`;
|
|
305
|
+
const file = existing.length === 1 ? existing[0] : canonicalFile;
|
|
306
|
+
const target = path.join(out, file);
|
|
307
|
+
// encodeURIComponent cannot emit a separator, but assert the boundary
|
|
308
|
+
// anyway: this writes to a git repo from a remote payload.
|
|
309
|
+
if (path.dirname(path.resolve(target)) !== out) {
|
|
310
|
+
throw new Error(`refusing to write block "${key}" outside ${out}`);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
const secrets = findPlaintextSecrets(block);
|
|
314
|
+
for (const at of secrets) report.plaintextSecrets.push(`${key}.${at}`);
|
|
315
|
+
|
|
316
|
+
const duplicates = existing.filter((f) => f !== file);
|
|
317
|
+
if (!opts.dryRun) {
|
|
318
|
+
for (const dup of duplicates) fs.rmSync(path.join(out, dup));
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
const next = serialize(block);
|
|
322
|
+
const current = fs.existsSync(target) ? fs.readFileSync(target, "utf-8") : null;
|
|
323
|
+
let currentParsed: unknown;
|
|
324
|
+
try {
|
|
325
|
+
currentParsed = current === null ? undefined : JSON.parse(current);
|
|
326
|
+
} catch {
|
|
327
|
+
currentParsed = undefined; // unparseable local file — overwrite it
|
|
328
|
+
}
|
|
329
|
+
const sameContent =
|
|
330
|
+
current !== null && stableStringify(currentParsed) === stableStringify(block);
|
|
331
|
+
if (sameContent && duplicates.length === 0) {
|
|
332
|
+
report.unchanged++;
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
if (!opts.dryRun) fs.writeFileSync(target, next);
|
|
336
|
+
(current === null ? report.added : report.updated).push(key);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
if (opts.prune) {
|
|
340
|
+
for (const [key, files] of existingByKey) {
|
|
341
|
+
if (seen.has(key)) continue;
|
|
342
|
+
if (deny.some((p) => matchesGlob(key, p))) {
|
|
343
|
+
report.denied.push(key);
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
// A local-only block holding credentials is site-owned (the migration put
|
|
347
|
+
// them there); upstream never had it, so its absence must not delete it.
|
|
348
|
+
if (!opts.allowSecretBlocks) {
|
|
349
|
+
const local = files.map((f) => {
|
|
350
|
+
try {
|
|
351
|
+
return JSON.parse(fs.readFileSync(path.join(out, f), "utf-8")) as unknown;
|
|
352
|
+
} catch {
|
|
353
|
+
return null;
|
|
354
|
+
}
|
|
355
|
+
});
|
|
356
|
+
if (local.some(hasEncryptedSecretRef)) {
|
|
357
|
+
report.protectedSecretBlocks.push(key);
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
if (!opts.dryRun) {
|
|
362
|
+
for (const f of files) fs.rmSync(path.join(out, f));
|
|
363
|
+
}
|
|
364
|
+
report.removed.push(key);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
return report;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// -------------------------------------------------------------------- CLI
|
|
372
|
+
|
|
373
|
+
interface CliOptions extends PullOptions {
|
|
374
|
+
url?: string;
|
|
375
|
+
maxBytes: number;
|
|
376
|
+
timeoutMs: number;
|
|
377
|
+
failOnPlaintextSecret: boolean;
|
|
378
|
+
json: boolean;
|
|
379
|
+
github: boolean;
|
|
380
|
+
help: boolean;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
function parseArgs(argv: string[]): CliOptions {
|
|
384
|
+
const opts: CliOptions = {
|
|
385
|
+
out: ".deco/blocks",
|
|
386
|
+
maxBytes: DEFAULT_MAX_BYTES,
|
|
387
|
+
timeoutMs: DEFAULT_TIMEOUT_MS,
|
|
388
|
+
failOnPlaintextSecret: false,
|
|
389
|
+
json: false,
|
|
390
|
+
github: false,
|
|
391
|
+
help: false,
|
|
392
|
+
};
|
|
393
|
+
for (let i = 0; i < argv.length; i++) {
|
|
394
|
+
const flag = argv[i];
|
|
395
|
+
switch (flag) {
|
|
396
|
+
case "--url":
|
|
397
|
+
opts.url = argv[++i];
|
|
398
|
+
break;
|
|
399
|
+
case "--origin": {
|
|
400
|
+
const origin = (argv[++i] ?? "").replace(/\/+$/, "");
|
|
401
|
+
opts.url = origin ? `${origin}/.decofile` : undefined;
|
|
402
|
+
break;
|
|
403
|
+
}
|
|
404
|
+
case "--out":
|
|
405
|
+
opts.out = argv[++i] ?? opts.out;
|
|
406
|
+
break;
|
|
407
|
+
case "--deny":
|
|
408
|
+
opts.deny = (argv[++i] ?? "")
|
|
409
|
+
.split(",")
|
|
410
|
+
.map((s) => s.trim())
|
|
411
|
+
.filter(Boolean);
|
|
412
|
+
break;
|
|
413
|
+
case "--allow-secret-blocks":
|
|
414
|
+
opts.allowSecretBlocks = true;
|
|
415
|
+
break;
|
|
416
|
+
case "--prune":
|
|
417
|
+
opts.prune = true;
|
|
418
|
+
break;
|
|
419
|
+
case "--dry-run":
|
|
420
|
+
opts.dryRun = true;
|
|
421
|
+
break;
|
|
422
|
+
case "--fail-on-plaintext-secret":
|
|
423
|
+
opts.failOnPlaintextSecret = true;
|
|
424
|
+
break;
|
|
425
|
+
case "--max-bytes":
|
|
426
|
+
opts.maxBytes = Number(argv[++i]);
|
|
427
|
+
break;
|
|
428
|
+
case "--timeout-ms":
|
|
429
|
+
opts.timeoutMs = Number(argv[++i]);
|
|
430
|
+
break;
|
|
431
|
+
case "--json":
|
|
432
|
+
opts.json = true;
|
|
433
|
+
break;
|
|
434
|
+
case "--github":
|
|
435
|
+
opts.github = true;
|
|
436
|
+
break;
|
|
437
|
+
case "--help":
|
|
438
|
+
case "-h":
|
|
439
|
+
opts.help = true;
|
|
440
|
+
break;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
return opts;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function showHelp(): void {
|
|
447
|
+
console.log(`
|
|
448
|
+
@decocms/blocks-cli — pull the production decofile into .deco/blocks/
|
|
449
|
+
|
|
450
|
+
Usage:
|
|
451
|
+
tsx sync-blocks-bot.ts --origin https://www.minhaloja.com.br [options]
|
|
452
|
+
|
|
453
|
+
Options:
|
|
454
|
+
--origin <url> Site origin; fetches <origin>/.decofile
|
|
455
|
+
--url <url> Full decofile URL (alternative to --origin)
|
|
456
|
+
--out <dir> Blocks directory (default: .deco/blocks)
|
|
457
|
+
--deny <globs> Comma-separated key globs never overwritten
|
|
458
|
+
(default: ${DEFAULT_DENY.join(",")})
|
|
459
|
+
--allow-secret-blocks Also overwrite blocks holding encrypted secrets
|
|
460
|
+
--prune Delete local blocks absent upstream
|
|
461
|
+
--dry-run Report only, write nothing
|
|
462
|
+
--fail-on-plaintext-secret Exit 1 if a written block holds a raw credential
|
|
463
|
+
--max-bytes <n> Payload cap (default: ${DEFAULT_MAX_BYTES})
|
|
464
|
+
--timeout-ms <n> Fetch timeout (default: ${DEFAULT_TIMEOUT_MS})
|
|
465
|
+
--json Emit the report as JSON
|
|
466
|
+
--github Emit ::notice::/::error:: lines for Actions
|
|
467
|
+
--help, -h This message
|
|
468
|
+
|
|
469
|
+
Exit codes:
|
|
470
|
+
0 done 1 plaintext secret gate 2 usage/network error
|
|
471
|
+
`);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
function reportToText(report: PullReport): string {
|
|
475
|
+
const lines = [
|
|
476
|
+
`remote blocks: ${report.remoteBlocks}${report.bytes ? ` (${report.bytes} bytes)` : ""}${report.revision ? ` revision ${report.revision}` : ""}`,
|
|
477
|
+
`added ${report.added.length} updated ${report.updated.length} unchanged ${report.unchanged} removed ${report.removed.length}`,
|
|
478
|
+
`denied ${report.denied.length} secret-protected ${report.protectedSecretBlocks.length} skipped ${report.skipped.length}`,
|
|
479
|
+
];
|
|
480
|
+
for (const [label, keys] of [
|
|
481
|
+
["added", report.added],
|
|
482
|
+
["updated", report.updated],
|
|
483
|
+
["removed", report.removed],
|
|
484
|
+
] as const) {
|
|
485
|
+
for (const key of keys.slice(0, 50)) lines.push(` ${label}: ${key}`);
|
|
486
|
+
if (keys.length > 50) lines.push(` ${label}: … and ${keys.length - 50} more`);
|
|
487
|
+
}
|
|
488
|
+
return lines.join("\n");
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
async function main(): Promise<void> {
|
|
492
|
+
const opts = parseArgs(process.argv.slice(2));
|
|
493
|
+
if (opts.help) {
|
|
494
|
+
showHelp();
|
|
495
|
+
process.exit(0);
|
|
496
|
+
}
|
|
497
|
+
if (!opts.url) {
|
|
498
|
+
console.error("sync-blocks-bot: --origin or --url is required (see --help)");
|
|
499
|
+
process.exit(2);
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
let fetched: FetchResult;
|
|
503
|
+
try {
|
|
504
|
+
fetched = await fetchDecofile(opts.url, { maxBytes: opts.maxBytes, timeoutMs: opts.timeoutMs });
|
|
505
|
+
} catch (e) {
|
|
506
|
+
console.error(`sync-blocks-bot: ${(e as Error).message}`);
|
|
507
|
+
process.exit(2);
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
let report: PullReport;
|
|
511
|
+
try {
|
|
512
|
+
report = writeDecofileToDir(fetched.blocks, opts);
|
|
513
|
+
} catch (e) {
|
|
514
|
+
console.error(`sync-blocks-bot: ${(e as Error).message}`);
|
|
515
|
+
process.exit(2);
|
|
516
|
+
}
|
|
517
|
+
report.revision = fetched.revision;
|
|
518
|
+
report.bytes = fetched.bytes;
|
|
519
|
+
|
|
520
|
+
process.stdout.write(
|
|
521
|
+
`${opts.json ? JSON.stringify({ url: opts.url, ...report }, null, 2) : reportToText(report)}\n`,
|
|
522
|
+
);
|
|
523
|
+
|
|
524
|
+
if (opts.github) {
|
|
525
|
+
process.stdout.write(
|
|
526
|
+
`::notice::decofile sync — +${report.added.length} ~${report.updated.length} -${report.removed.length} (${report.remoteBlocks} blocks upstream)\n`,
|
|
527
|
+
);
|
|
528
|
+
for (const at of report.plaintextSecrets) {
|
|
529
|
+
process.stdout.write(
|
|
530
|
+
`::error title=plaintext-secret::${at} looks like a credential in the clear\n`,
|
|
531
|
+
);
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
if (report.plaintextSecrets.length > 0 && opts.failOnPlaintextSecret) {
|
|
536
|
+
console.error(
|
|
537
|
+
`sync-blocks-bot: ${report.plaintextSecrets.length} plaintext credential(s) in the pulled content — refusing to commit it. Move them to encrypted secrets upstream, or deny the block with --deny.`,
|
|
538
|
+
);
|
|
539
|
+
process.exit(1);
|
|
540
|
+
}
|
|
541
|
+
process.exit(0);
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* Entry detection must resolve symlinks. Invoked through the package `bin`
|
|
546
|
+
* (`npx --package=@decocms/blocks-cli deco-sync-blocks-bot`), `process.argv[1]`
|
|
547
|
+
* is the `node_modules/.bin/...` symlink while `import.meta.url` is the real
|
|
548
|
+
* file — a plain string compare fails, `main()` never runs, and the CLI exits 0
|
|
549
|
+
* having printed nothing. Observed on a real CI run; the workflow's report
|
|
550
|
+
* assertion is what surfaced it.
|
|
551
|
+
*/
|
|
552
|
+
function isMainModule(): boolean {
|
|
553
|
+
if (typeof require !== "undefined" && typeof module !== "undefined" && require.main === module) {
|
|
554
|
+
return true;
|
|
555
|
+
}
|
|
556
|
+
try {
|
|
557
|
+
const arg = process.argv?.[1];
|
|
558
|
+
if (!arg) return false;
|
|
559
|
+
return fs.realpathSync(arg) === fs.realpathSync(fileURLToPath(import.meta.url));
|
|
560
|
+
} catch {
|
|
561
|
+
return false;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
if (isMainModule()) {
|
|
566
|
+
void main();
|
|
567
|
+
}
|
|
@@ -46,11 +46,14 @@
|
|
|
46
46
|
|
|
47
47
|
import { execSync } from "node:child_process";
|
|
48
48
|
import * as path from "node:path";
|
|
49
|
+
import { redirectPrefix } from "@decocms/blocks/cms";
|
|
50
|
+
import { splitExactRedirects } from "@decocms/blocks/sdk/redirects";
|
|
49
51
|
import { createKvRestClient, kvConfigFromEnv } from "./lib/cf-kv-rest";
|
|
50
52
|
import {
|
|
51
53
|
buildSnapshot,
|
|
52
54
|
recordAndGcDeployment,
|
|
53
55
|
setLiveDeployment,
|
|
56
|
+
syncRedirectsToKv,
|
|
54
57
|
verifySnapshotInKv,
|
|
55
58
|
writeSnapshotToKv,
|
|
56
59
|
} from "./lib/kv-snapshot";
|
|
@@ -160,12 +163,20 @@ async function main() {
|
|
|
160
163
|
process.exit(2);
|
|
161
164
|
}
|
|
162
165
|
|
|
163
|
-
|
|
166
|
+
// Exact redirects get ONE KV key each and are removed from the decofile. A
|
|
167
|
+
// bulk-migration site can carry tens of thousands; inside the snapshot they
|
|
168
|
+
// are resident in every isolate twice (parsed graph + RedirectMap) for data
|
|
169
|
+
// that usually matches nothing. Globs stay — they must be scanned in order.
|
|
170
|
+
const split = splitExactRedirects(blocks);
|
|
164
171
|
const purgePaths = opts.all ? ["/"] : purgePathsForChangedKeys(blocks, changedKeys);
|
|
172
|
+
const snap = buildSnapshot(split.blocks);
|
|
165
173
|
console.log(`decofile: ${snap.count} blocks, revision ${snap.revision} → deployment ${opts.deploymentId}`);
|
|
174
|
+
if (split.exact.length > 0) {
|
|
175
|
+
console.log(`redirects: ${split.exact.length} exact rule(s) → ${redirectPrefix(opts.deploymentId)}*`);
|
|
176
|
+
}
|
|
166
177
|
|
|
167
178
|
if (!opts.write) {
|
|
168
|
-
console.log(`\nDry-run only. Would write decofile:${opts.deploymentId} + revision, GC to ${opts.retain}, purge: ${purgePaths.join(", ")}`);
|
|
179
|
+
console.log(`\nDry-run only. Would write decofile:${opts.deploymentId} + revision, ${split.exact.length} redirect key(s), GC to ${opts.retain}, purge: ${purgePaths.join(", ")}`);
|
|
169
180
|
process.exit(0);
|
|
170
181
|
}
|
|
171
182
|
|
|
@@ -178,6 +189,13 @@ async function main() {
|
|
|
178
189
|
}
|
|
179
190
|
|
|
180
191
|
try {
|
|
192
|
+
// Redirects FIRST: the decofile no longer carries them, so publishing the
|
|
193
|
+
// snapshot before the keys exist would leave a window where every migrated
|
|
194
|
+
// URL 404s instead of redirecting.
|
|
195
|
+
const red = await syncRedirectsToKv(client, split.exact, opts.deploymentId);
|
|
196
|
+
if (red.written || red.deleted) {
|
|
197
|
+
console.log(`redirects: ${red.written} written, ${red.deleted} removed.`);
|
|
198
|
+
}
|
|
181
199
|
await writeSnapshotToKv(client, snap, opts.deploymentId);
|
|
182
200
|
const verify = await verifySnapshotInKv(client, snap.revision, opts.deploymentId);
|
|
183
201
|
if (!verify.ok) {
|
|
File without changes
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Generator default output paths flipped from `src/server/{cms,admin}/` to
|
|
3
|
-
* `.deco/` (framework artifacts live in the framework's folder, not mixed
|
|
4
|
-
* into app source). Sites that never pass an explicit `--out`/`--out-file`
|
|
5
|
-
* flag pick up the new default silently — except when the OLD default file
|
|
6
|
-
* is still sitting on disk, which almost always means something (an
|
|
7
|
-
* importer, a `.gitignore` entry, a stale CI cache check) still points at
|
|
8
|
-
* it. In that case we warn once, to stderr, and then write to the NEW
|
|
9
|
-
* default anyway: the artifact is regenerated code, so there's no reason to
|
|
10
|
-
* block the run — the warning is just a nudge to go clean up the stale file
|
|
11
|
-
* and its importers.
|
|
12
|
-
*
|
|
13
|
-
* An explicit flag means the caller made a deliberate choice about where
|
|
14
|
-
* output goes; it gets no warning and no guard.
|
|
15
|
-
*/
|
|
16
|
-
export function warnLegacyArtifact(oldPath: string, newPath: string): void {
|
|
17
|
-
console.warn(
|
|
18
|
-
`[deco] Generator default output moved: ${oldPath} -> ${newPath}. Move the file and update its importers.`,
|
|
19
|
-
);
|
|
20
|
-
}
|