@decocms/blocks-cli 7.49.0 → 7.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -2
- package/scripts/migrate/phase-scaffold.ts +10 -0
- package/scripts/migrate/phase-verify.ts +1 -0
- package/scripts/migrate/templates/ci-workflows.test.ts +43 -0
- package/scripts/migrate/templates/content-sync-yml.ts +184 -0
- package/scripts/pull-decofile.test.ts +258 -0
- package/scripts/pull-decofile.ts +557 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@decocms/blocks-cli",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.50.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Deco codegen (generate-blocks, generate-schema, generate-invoke) and Fresh-to-TanStack migration tooling",
|
|
6
6
|
"repository": {
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
"deco-audit-observability": "./scripts/audit-observability-config.ts",
|
|
18
18
|
"deco-migrate-blocks-to-kv": "./scripts/migrate-blocks-to-kv.ts",
|
|
19
19
|
"deco-sync-blocks-to-kv": "./scripts/sync-blocks-to-kv.ts",
|
|
20
|
+
"deco-pull-decofile": "./scripts/pull-decofile.ts",
|
|
20
21
|
"deco-upgrade-6-to-7": "./scripts/upgrade-6-to-7.ts",
|
|
21
22
|
"deco-reconcile": "./scripts/reconcile.ts"
|
|
22
23
|
},
|
|
@@ -32,7 +33,7 @@
|
|
|
32
33
|
"lint:unused": "knip"
|
|
33
34
|
},
|
|
34
35
|
"dependencies": {
|
|
35
|
-
"@decocms/blocks": "7.
|
|
36
|
+
"@decocms/blocks": "7.50.0",
|
|
36
37
|
"ts-morph": "^27.0.0",
|
|
37
38
|
"tsx": "^4.22.5"
|
|
38
39
|
},
|
|
@@ -9,6 +9,7 @@ import { generateCacheConfig } from "./templates/cache-config";
|
|
|
9
9
|
import { generateCiFiles } from "./templates/ci-yml";
|
|
10
10
|
import { generateCommerceInit } from "./templates/commerce-init";
|
|
11
11
|
import { generateCommerceLoaders } from "./templates/commerce-loaders";
|
|
12
|
+
import { generateContentSyncYml } from "./templates/content-sync-yml";
|
|
12
13
|
import { generateMigrationPolicyPointerRule } from "./templates/cursor-rules";
|
|
13
14
|
import { generateHooks } from "./templates/hooks";
|
|
14
15
|
import { generateKnipConfig } from "./templates/knip-config";
|
|
@@ -130,6 +131,15 @@ export function scaffold(ctx: MigrationContext): void {
|
|
|
130
131
|
// live storefront (@decocms/parity). Inert until PARITY_PROD_URL is set.
|
|
131
132
|
writeFile(ctx, ".github/workflows/parity.yml", generateParityYml(ctx.siteName));
|
|
132
133
|
|
|
134
|
+
// Daily content pull from the still-live storefront (`<origin>/.decofile` ->
|
|
135
|
+
// `.deco/blocks` -> PR). Replaces the legacy cross-repo-PAT push sync; inert
|
|
136
|
+
// until the operator sets the repo variable CONTENT_SYNC_ORIGIN.
|
|
137
|
+
writeFile(
|
|
138
|
+
ctx,
|
|
139
|
+
".github/workflows/content-sync.yml",
|
|
140
|
+
generateContentSyncYml(CANONICAL_BUN_VERSION),
|
|
141
|
+
);
|
|
142
|
+
|
|
133
143
|
// Server entry files (server.ts, worker-entry.ts, router.tsx, runtime.ts, context.ts)
|
|
134
144
|
writeMultiFile(ctx, generateServerEntry(ctx));
|
|
135
145
|
|
|
@@ -26,6 +26,7 @@ const REQUIRED_FILES = [
|
|
|
26
26
|
".github/workflows/playwright.yml",
|
|
27
27
|
".github/workflows/react-doctor.yml",
|
|
28
28
|
".github/workflows/parity.yml",
|
|
29
|
+
".github/workflows/content-sync.yml",
|
|
29
30
|
"tools/gates/no-suppressions.sh",
|
|
30
31
|
"playwright.config.ts",
|
|
31
32
|
"knip.config.ts",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
2
|
import { generateCiFiles } from "./ci-yml";
|
|
3
|
+
import { generateContentSyncYml } from "./content-sync-yml";
|
|
3
4
|
import { generateMainPushGuardYml } from "./main-push-guard-yml";
|
|
4
5
|
import { generateParityYml } from "./parity-yml";
|
|
5
6
|
import { generatePlaywrightFiles } from "./playwright-yml";
|
|
@@ -123,3 +124,45 @@ describe("generateParityYml", () => {
|
|
|
123
124
|
expect(yml).toContain("secrets.ANTHROPIC_API_KEY"); // optional LLM key
|
|
124
125
|
});
|
|
125
126
|
});
|
|
127
|
+
|
|
128
|
+
describe("generateContentSyncYml", () => {
|
|
129
|
+
const yml = generateContentSyncYml("bun@1.3.5");
|
|
130
|
+
|
|
131
|
+
it("pulls on a daily cron, gated on CONTENT_SYNC_ORIGIN", () => {
|
|
132
|
+
expect(yml).toContain("name: content-sync");
|
|
133
|
+
expect(yml).toContain('BUN_VERSION: "1.3.5"'); // bun@ prefix stripped
|
|
134
|
+
expect(yml).toContain('- cron: "0 6 * * *"');
|
|
135
|
+
expect(yml).toContain("vars.CONTENT_SYNC_ORIGIN");
|
|
136
|
+
expect(yml).toContain("scripts/pull-decofile.ts");
|
|
137
|
+
expect(yml).toContain("--fail-on-plaintext-secret");
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it("only ever needs GITHUB_TOKEN — no cross-repo credential", () => {
|
|
141
|
+
expect(yml).toContain("secrets.GITHUB_TOKEN");
|
|
142
|
+
expect(yml).not.toMatch(/secrets\.(?!GITHUB_TOKEN)[A-Z_]+/);
|
|
143
|
+
expect(yml).toContain("permissions:");
|
|
144
|
+
expect(yml).toContain("concurrency:");
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
it("guards the diff to .deco/blocks and builds before opening the PR", () => {
|
|
148
|
+
const guardIdx = yml.indexOf("Guard");
|
|
149
|
+
const buildIdx = yml.indexOf("Validar (generate + build)");
|
|
150
|
+
const prIdx = yml.indexOf("gh pr create");
|
|
151
|
+
expect(guardIdx).toBeGreaterThan(-1);
|
|
152
|
+
expect(guardIdx).toBeLessThan(buildIdx);
|
|
153
|
+
expect(buildIdx).toBeLessThan(prIdx);
|
|
154
|
+
expect(yml).toContain("grep -v '^\\.deco/blocks/'");
|
|
155
|
+
expect(yml).toContain("git add .deco/blocks");
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
it("is de-projectized — no real site/customer names", () => {
|
|
159
|
+
expect(yml).not.toMatch(/oficina|miess|colombo/i);
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
it("runs the site's own installed CLI, or a pinned one when given", () => {
|
|
163
|
+
expect(yml).toContain("bunx tsx node_modules/@decocms/blocks-cli/scripts/pull-decofile.ts");
|
|
164
|
+
const pinned = generateContentSyncYml("1.3.5", "7.50.0");
|
|
165
|
+
expect(pinned).toContain("bunx -y @decocms/blocks-cli@7.50.0 deco-pull-decofile");
|
|
166
|
+
expect(pinned).not.toContain("node_modules/@decocms/blocks-cli");
|
|
167
|
+
});
|
|
168
|
+
});
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scaffolds `.github/workflows/content-sync.yml` — the secure content channel
|
|
3
|
+
* from the still-live Fresh/Deno storefront into the migrated repo.
|
|
4
|
+
*
|
|
5
|
+
* Replaces the legacy push-based sync (a workflow in the *legacy* repo holding a
|
|
6
|
+
* cross-repo PAT, `rsync --delete` + `git push` straight into this repo's main).
|
|
7
|
+
* That token is a code-write door, not a content channel: whoever controls the
|
|
8
|
+
* legacy repo or the token can land arbitrary files here, unreviewed.
|
|
9
|
+
*
|
|
10
|
+
* Direction is inverted — this repo pulls, on its own schedule, with its own
|
|
11
|
+
* `GITHUB_TOKEN`, and gates the result:
|
|
12
|
+
* - `pull-decofile.ts` fetches `<origin>/.decofile` (public, unauthenticated)
|
|
13
|
+
* and writes `.deco/blocks/`, denying the `Site` block and anything holding
|
|
14
|
+
* encrypted credentials, and aborting on a plaintext credential;
|
|
15
|
+
* - a path guard fails the run if anything outside `.deco/blocks/` changed;
|
|
16
|
+
* - `bun run generate && bun run build` runs IN THIS JOB, before the PR. It
|
|
17
|
+
* has to be here: a PR opened with `GITHUB_TOKEN` does not trigger
|
|
18
|
+
* `pull_request` workflows, so gating on the PR's own CI would need a PAT
|
|
19
|
+
* and would never fire on its own.
|
|
20
|
+
*
|
|
21
|
+
* Inert until the operator sets the repo variable `CONTENT_SYNC_ORIGIN` (same
|
|
22
|
+
* one-knob pattern as parity.yml's `PARITY_PROD_URL`) — the job skips cleanly.
|
|
23
|
+
*
|
|
24
|
+
* Docs: docs/content-sync.md
|
|
25
|
+
*
|
|
26
|
+
* @param bunVersion Pinned bun version, in lockstep with package.json.
|
|
27
|
+
* @param cliVersion Optional `@decocms/blocks-cli` version to run the pull
|
|
28
|
+
* with, via `bunx`. Omit for a freshly scaffolded site: the site's own
|
|
29
|
+
* installed copy is by definition in-version, so the local file path is used.
|
|
30
|
+
* Pass a version for a site still on an older `@decocms/*` — pinning only the
|
|
31
|
+
* sync step gets the script without bumping the runtime the site builds
|
|
32
|
+
* against (blocks-cli pins `@decocms/blocks` exactly, so bumping the devDep
|
|
33
|
+
* drags a second runtime version into the tree). Drop it when the site bumps.
|
|
34
|
+
*/
|
|
35
|
+
export function generateContentSyncYml(bunVersion: string, cliVersion?: string): string {
|
|
36
|
+
const bun = bunVersion.replace(/^bun@/, "");
|
|
37
|
+
const pullCmd = cliVersion
|
|
38
|
+
? `bunx -y @decocms/blocks-cli@${cliVersion} deco-pull-decofile`
|
|
39
|
+
: "bunx tsx node_modules/@decocms/blocks-cli/scripts/pull-decofile.ts";
|
|
40
|
+
return `name: content-sync
|
|
41
|
+
|
|
42
|
+
# Puxa o conteúdo publicado na loja de produção (Fresh/Deno) para \`.deco/blocks\`
|
|
43
|
+
# e abre um PR. Sem token cross-repo: o repo legado não tem permissão nenhuma
|
|
44
|
+
# aqui — este repo busca sozinho, valida e só então mergeia.
|
|
45
|
+
#
|
|
46
|
+
# Configuração (uma vez): variável de repo \`CONTENT_SYNC_ORIGIN\` = origin da
|
|
47
|
+
# loja de produção (ex.: https://www.minhaloja.com.br). Sem ela o job pula limpo.
|
|
48
|
+
# Para revisão humana em vez de merge automático, mude \`AUTO_MERGE\` para "false".
|
|
49
|
+
#
|
|
50
|
+
# Depois de ligar isto, APAGUE o workflow de push no repo legado e REVOGUE o
|
|
51
|
+
# token cross-repo — o pull não fecha aquela porta sozinho. Ver docs/content-sync.md.
|
|
52
|
+
|
|
53
|
+
on:
|
|
54
|
+
schedule:
|
|
55
|
+
# 06:00 UTC = 03:00 BRT, fora do horário de publicação do CMS.
|
|
56
|
+
- cron: "0 6 * * *"
|
|
57
|
+
workflow_dispatch:
|
|
58
|
+
inputs:
|
|
59
|
+
origin:
|
|
60
|
+
description: "Origin da loja de produção (sobrepõe CONTENT_SYNC_ORIGIN)"
|
|
61
|
+
required: false
|
|
62
|
+
prune:
|
|
63
|
+
description: "Apagar blocos que não existem mais em produção"
|
|
64
|
+
type: boolean
|
|
65
|
+
default: true
|
|
66
|
+
dry_run:
|
|
67
|
+
description: "Só relatório, não escreve nem abre PR"
|
|
68
|
+
type: boolean
|
|
69
|
+
default: false
|
|
70
|
+
|
|
71
|
+
permissions:
|
|
72
|
+
contents: write
|
|
73
|
+
pull-requests: write
|
|
74
|
+
|
|
75
|
+
# Publicações do CMS acontecem em rajada; uma sync por vez, sem cancelar a que
|
|
76
|
+
# já está no gate de build.
|
|
77
|
+
concurrency:
|
|
78
|
+
group: content-sync
|
|
79
|
+
cancel-in-progress: false
|
|
80
|
+
|
|
81
|
+
env:
|
|
82
|
+
BUN_VERSION: "${bun}"
|
|
83
|
+
AUTO_MERGE: "true"
|
|
84
|
+
# Chaves de bloco que a sync NUNCA sobrescreve. Blocos com secret encriptado
|
|
85
|
+
# já são protegidos por shape pelo script (as credenciais deste repo vivem no
|
|
86
|
+
# bloco de app dele, que não existe nesse layout em produção).
|
|
87
|
+
DENY_KEYS: "Site,site"
|
|
88
|
+
|
|
89
|
+
jobs:
|
|
90
|
+
sync:
|
|
91
|
+
runs-on: ubuntu-latest
|
|
92
|
+
steps:
|
|
93
|
+
- name: Resolve origin
|
|
94
|
+
id: cfg
|
|
95
|
+
env:
|
|
96
|
+
INPUT_ORIGIN: \${{ github.event.inputs.origin }}
|
|
97
|
+
VAR_ORIGIN: \${{ vars.CONTENT_SYNC_ORIGIN }}
|
|
98
|
+
run: |
|
|
99
|
+
origin="\${INPUT_ORIGIN:-$VAR_ORIGIN}"
|
|
100
|
+
if [ -z "$origin" ]; then
|
|
101
|
+
echo "::notice::variável de repo CONTENT_SYNC_ORIGIN não configurada — content-sync inerte."
|
|
102
|
+
echo "skip=true" >> "$GITHUB_OUTPUT"
|
|
103
|
+
else
|
|
104
|
+
echo "origin=$origin" >> "$GITHUB_OUTPUT"
|
|
105
|
+
fi
|
|
106
|
+
|
|
107
|
+
- uses: actions/checkout@v4
|
|
108
|
+
if: steps.cfg.outputs.skip != 'true'
|
|
109
|
+
|
|
110
|
+
- uses: oven-sh/setup-bun@v2
|
|
111
|
+
if: steps.cfg.outputs.skip != 'true'
|
|
112
|
+
with:
|
|
113
|
+
bun-version: \${{ env.BUN_VERSION }}
|
|
114
|
+
|
|
115
|
+
- name: Install
|
|
116
|
+
if: steps.cfg.outputs.skip != 'true'
|
|
117
|
+
run: bun install --frozen-lockfile
|
|
118
|
+
|
|
119
|
+
- name: Pull decofile de produção
|
|
120
|
+
if: steps.cfg.outputs.skip != 'true'
|
|
121
|
+
env:
|
|
122
|
+
ORIGIN: \${{ steps.cfg.outputs.origin }}
|
|
123
|
+
PRUNE: \${{ github.event.inputs.prune == 'false' && ' ' || '--prune' }}
|
|
124
|
+
DRY_RUN: \${{ github.event.inputs.dry_run == 'true' && '--dry-run' || ' ' }}
|
|
125
|
+
run: |
|
|
126
|
+
${pullCmd} \\
|
|
127
|
+
--origin "$ORIGIN" \\
|
|
128
|
+
--out .deco/blocks \\
|
|
129
|
+
--deny "$DENY_KEYS" \\
|
|
130
|
+
--fail-on-plaintext-secret \\
|
|
131
|
+
--github $PRUNE $DRY_RUN
|
|
132
|
+
|
|
133
|
+
- name: Guard — só .deco/blocks pode mudar
|
|
134
|
+
id: guard
|
|
135
|
+
if: steps.cfg.outputs.skip != 'true' && github.event.inputs.dry_run != 'true'
|
|
136
|
+
run: |
|
|
137
|
+
{
|
|
138
|
+
git -c core.quotepath=false diff --name-only HEAD
|
|
139
|
+
git -c core.quotepath=false ls-files --others --exclude-standard
|
|
140
|
+
} | sort -u > /tmp/content-sync-changed.txt
|
|
141
|
+
offending="$(grep -v '^\\.deco/blocks/' /tmp/content-sync-changed.txt || true)"
|
|
142
|
+
if [ -n "$offending" ]; then
|
|
143
|
+
echo "::error::a sync mexeu fora de .deco/blocks — abortando:"
|
|
144
|
+
echo "$offending"
|
|
145
|
+
exit 1
|
|
146
|
+
fi
|
|
147
|
+
if [ ! -s /tmp/content-sync-changed.txt ]; then
|
|
148
|
+
echo "::notice::conteúdo já está em dia, nada a sincronizar."
|
|
149
|
+
echo "changed=false" >> "$GITHUB_OUTPUT"
|
|
150
|
+
else
|
|
151
|
+
echo "changed=true" >> "$GITHUB_OUTPUT"
|
|
152
|
+
echo "$(wc -l < /tmp/content-sync-changed.txt) arquivo(s) de bloco alterado(s)"
|
|
153
|
+
fi
|
|
154
|
+
|
|
155
|
+
# Gate de verdade. Roda AQUI porque PR aberto com GITHUB_TOKEN não dispara
|
|
156
|
+
# o workflow de \`pull_request\` — gatear no CI do PR exigiria um PAT.
|
|
157
|
+
- name: Validar (generate + build)
|
|
158
|
+
if: steps.guard.outputs.changed == 'true'
|
|
159
|
+
run: bun run generate && bun run build
|
|
160
|
+
|
|
161
|
+
- name: Abrir PR (e mergear se AUTO_MERGE)
|
|
162
|
+
if: steps.guard.outputs.changed == 'true'
|
|
163
|
+
env:
|
|
164
|
+
GH_TOKEN: \${{ secrets.GITHUB_TOKEN }}
|
|
165
|
+
ORIGIN: \${{ steps.cfg.outputs.origin }}
|
|
166
|
+
run: |
|
|
167
|
+
branch="content-sync/$(date -u +%Y-%m-%dT%H%M%SZ)"
|
|
168
|
+
git config user.name "github-actions[bot]"
|
|
169
|
+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
170
|
+
git checkout -b "$branch"
|
|
171
|
+
git add .deco/blocks
|
|
172
|
+
git commit -m "chore(content): sync .deco/blocks de $ORIGIN"
|
|
173
|
+
git push origin "$branch"
|
|
174
|
+
url="$(gh pr create \\
|
|
175
|
+
--title "chore(content): sync .deco/blocks de produção" \\
|
|
176
|
+
--body "Conteúdo puxado de \\\`$ORIGIN/.decofile\\\` pelo workflow \\\`content-sync\\\`. Só \\\`.deco/blocks/**\\\` mudou (guard) e \\\`generate + build\\\` passou antes deste PR existir." \\
|
|
177
|
+
--head "$branch")"
|
|
178
|
+
echo "PR: $url"
|
|
179
|
+
if [ "$AUTO_MERGE" = "true" ]; then
|
|
180
|
+
gh pr merge "$url" --squash --delete-branch || \\
|
|
181
|
+
echo "::notice::merge automático bloqueado (branch protection?) — PR aberto para revisão: $url"
|
|
182
|
+
fi
|
|
183
|
+
`;
|
|
184
|
+
}
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as os from "node:os";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
5
|
+
import {
|
|
6
|
+
DEFAULT_DENY,
|
|
7
|
+
fetchDecofile,
|
|
8
|
+
findPlaintextSecrets,
|
|
9
|
+
hasEncryptedSecretRef,
|
|
10
|
+
matchesGlob,
|
|
11
|
+
writeDecofileToDir,
|
|
12
|
+
} from "./pull-decofile";
|
|
13
|
+
|
|
14
|
+
function tmpBlocksDir(files: Record<string, unknown> = {}): string {
|
|
15
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "pull-decofile-"));
|
|
16
|
+
const out = path.join(dir, ".deco", "blocks");
|
|
17
|
+
fs.mkdirSync(out, { recursive: true });
|
|
18
|
+
for (const [file, content] of Object.entries(files)) {
|
|
19
|
+
fs.writeFileSync(path.join(out, file), `${JSON.stringify(content, null, 2)}\n`);
|
|
20
|
+
}
|
|
21
|
+
return out;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const ls = (dir: string) => fs.readdirSync(dir).sort();
|
|
25
|
+
const read = (dir: string, file: string) =>
|
|
26
|
+
JSON.parse(fs.readFileSync(path.join(dir, file), "utf-8")) as Record<string, unknown>;
|
|
27
|
+
|
|
28
|
+
describe("matchesGlob", () => {
|
|
29
|
+
it("matches whole keys with a * wildcard", () => {
|
|
30
|
+
expect(matchesGlob("Site", "Site")).toBe(true);
|
|
31
|
+
expect(matchesGlob("Sitemap", "Site")).toBe(false);
|
|
32
|
+
expect(matchesGlob("pages-Home", "pages-*")).toBe(true);
|
|
33
|
+
expect(matchesGlob("deco-vtex", "deco-*")).toBe(true);
|
|
34
|
+
// dots are literal, not "any char"
|
|
35
|
+
expect(matchesGlob("axb", "a.b")).toBe(false);
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
describe("hasEncryptedSecretRef", () => {
|
|
40
|
+
it("finds a {name, encrypted} ref at any depth", () => {
|
|
41
|
+
expect(hasEncryptedSecretRef({ appKey: { name: "BANANA", encrypted: "3a714de1" } })).toBe(true);
|
|
42
|
+
expect(hasEncryptedSecretRef({ a: [{ b: { name: "X", encrypted: "y" } }] })).toBe(true);
|
|
43
|
+
expect(hasEncryptedSecretRef({ name: "X", encrypted: "" })).toBe(false);
|
|
44
|
+
expect(hasEncryptedSecretRef({ sections: [{ __resolveType: "site/x.tsx" }] })).toBe(false);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
describe("findPlaintextSecrets", () => {
|
|
49
|
+
it("flags credential-shaped props holding a raw string", () => {
|
|
50
|
+
expect(findPlaintextSecrets({ appToken: "abcdefghijklmnop" })).toEqual(["appToken"]);
|
|
51
|
+
expect(findPlaintextSecrets({ nested: { api_key: "abcdefghijklmnop" } })).toEqual([
|
|
52
|
+
"nested.api_key",
|
|
53
|
+
]);
|
|
54
|
+
expect(findPlaintextSecrets({ list: [{ password: "abcdefghijklmnop" }] })).toEqual([
|
|
55
|
+
"list[0].password",
|
|
56
|
+
]);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it("ignores a bare `key` prop — CMS content is full of them", () => {
|
|
60
|
+
// real shape: every VTEX PLP loader block carries selectedFacets[].key
|
|
61
|
+
expect(
|
|
62
|
+
findPlaintextSecrets({ selectedFacets: [{ key: "productClusterIds", value: "140" }] }),
|
|
63
|
+
).toEqual([]);
|
|
64
|
+
// …but a qualified key still counts
|
|
65
|
+
expect(findPlaintextSecrets({ apiKey: "abcdefghijklmnop" })).toEqual(["apiKey"]);
|
|
66
|
+
expect(findPlaintextSecrets({ "private-key": "abcdefghijklmnop" })).toEqual(["private-key"]);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it("does not flag encrypted refs, urls, prose or short values", () => {
|
|
70
|
+
expect(findPlaintextSecrets({ appKey: { name: "X", encrypted: "abcdefghijklmnop" } })).toEqual(
|
|
71
|
+
[],
|
|
72
|
+
);
|
|
73
|
+
expect(findPlaintextSecrets({ tokenUrl: "https://x.com/very/long/path" })).toEqual([]);
|
|
74
|
+
expect(findPlaintextSecrets({ secret: "short" })).toEqual([]);
|
|
75
|
+
expect(findPlaintextSecrets({ password: "uma frase com espacos" })).toEqual([]);
|
|
76
|
+
expect(findPlaintextSecrets({ apiKey: "{{ FROM_ENV_VAR_HERE }}" })).toEqual([]);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
describe("writeDecofileToDir", () => {
|
|
81
|
+
it("writes one single-encoded file per block and skips unchanged ones", () => {
|
|
82
|
+
const out = tmpBlocksDir();
|
|
83
|
+
const remote = {
|
|
84
|
+
"pages-Home": { name: "Home", path: "/", sections: [] },
|
|
85
|
+
"pages-A B": { name: "A B", path: "/a-b", sections: [] },
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const first = writeDecofileToDir(remote, { out });
|
|
89
|
+
expect(first.added.sort()).toEqual(["pages-A B", "pages-Home"]);
|
|
90
|
+
expect(ls(out)).toEqual(["pages-A%20B.json", "pages-Home.json"]);
|
|
91
|
+
expect(read(out, "pages-Home.json").path).toBe("/");
|
|
92
|
+
|
|
93
|
+
const second = writeDecofileToDir(remote, { out });
|
|
94
|
+
expect(second.unchanged).toBe(2);
|
|
95
|
+
expect(second.added).toEqual([]);
|
|
96
|
+
expect(second.updated).toEqual([]);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it("compares content, not bytes — minified or reordered locals are not churn", () => {
|
|
100
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "pull-decofile-"));
|
|
101
|
+
const out = path.join(dir, "blocks");
|
|
102
|
+
fs.mkdirSync(out, { recursive: true });
|
|
103
|
+
// how the Studio daemon / old sync bot write it: one minified line, no newline
|
|
104
|
+
fs.writeFileSync(path.join(out, "pages-Home.json"), '{"sections":[],"path":"/","name":"Home"}');
|
|
105
|
+
|
|
106
|
+
const report = writeDecofileToDir(
|
|
107
|
+
{ "pages-Home": { name: "Home", path: "/", sections: [] } },
|
|
108
|
+
{ out },
|
|
109
|
+
);
|
|
110
|
+
expect(report.unchanged).toBe(1);
|
|
111
|
+
expect(report.updated).toEqual([]);
|
|
112
|
+
// untouched: no reformat commit
|
|
113
|
+
expect(fs.readFileSync(path.join(out, "pages-Home.json"), "utf-8")).toBe(
|
|
114
|
+
'{"sections":[],"path":"/","name":"Home"}',
|
|
115
|
+
);
|
|
116
|
+
|
|
117
|
+
// a real content change does get written, pretty-printed
|
|
118
|
+
const changed = writeDecofileToDir({ "pages-Home": { name: "Home v2", path: "/" } }, { out });
|
|
119
|
+
expect(changed.updated).toEqual(["pages-Home"]);
|
|
120
|
+
expect(fs.readFileSync(path.join(out, "pages-Home.json"), "utf-8")).toContain('\n "name"');
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it("denies keys by glob, defaulting to the Site block", () => {
|
|
124
|
+
const out = tmpBlocksDir({ "Site.json": { seo: { title: "local" } } });
|
|
125
|
+
const report = writeDecofileToDir(
|
|
126
|
+
{ Site: { seo: { title: "prod" } }, "pages-Home": { path: "/" } },
|
|
127
|
+
{ out },
|
|
128
|
+
);
|
|
129
|
+
expect(DEFAULT_DENY).toContain("Site");
|
|
130
|
+
expect(report.denied).toEqual(["Site"]);
|
|
131
|
+
expect(read(out, "Site.json").seo).toEqual({ title: "local" });
|
|
132
|
+
|
|
133
|
+
const custom = writeDecofileToDir({ "pages-Home": { path: "/x" } }, { out, deny: ["pages-*"] });
|
|
134
|
+
expect(custom.denied).toEqual(["pages-Home"]);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
it("never overwrites a block that carries an encrypted secret", () => {
|
|
138
|
+
const out = tmpBlocksDir({
|
|
139
|
+
"deco-vtex.json": { account: "loja", appKey: { name: "KEY", encrypted: "local" } },
|
|
140
|
+
});
|
|
141
|
+
const report = writeDecofileToDir(
|
|
142
|
+
{ "deco-vtex": { account: "outra", appKey: { name: "KEY", encrypted: "prod" } } },
|
|
143
|
+
{ out },
|
|
144
|
+
);
|
|
145
|
+
expect(report.protectedSecretBlocks).toEqual(["deco-vtex"]);
|
|
146
|
+
expect(read(out, "deco-vtex.json").account).toBe("loja");
|
|
147
|
+
|
|
148
|
+
const forced = writeDecofileToDir(
|
|
149
|
+
{ "deco-vtex": { account: "outra", appKey: { name: "KEY", encrypted: "prod" } } },
|
|
150
|
+
{ out, allowSecretBlocks: true },
|
|
151
|
+
);
|
|
152
|
+
expect(forced.updated).toEqual(["deco-vtex"]);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it("overwrites a legacy double-encoded filename in place instead of duplicating it", () => {
|
|
156
|
+
const out = tmpBlocksDir({ "pages-A%2520B.json": { name: "old", path: "/a-b" } });
|
|
157
|
+
const report = writeDecofileToDir({ "pages-A B": { name: "new", path: "/a-b" } }, { out });
|
|
158
|
+
expect(report.updated).toEqual(["pages-A B"]);
|
|
159
|
+
expect(ls(out)).toEqual(["pages-A%2520B.json"]);
|
|
160
|
+
expect(read(out, "pages-A%2520B.json").name).toBe("new");
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it("collapses colliding encodings onto the canonical name", () => {
|
|
164
|
+
const out = tmpBlocksDir({
|
|
165
|
+
"pages-A%2520B.json": { name: "stale", path: "/a-b" },
|
|
166
|
+
"pages-A%20B.json": { name: "also stale", path: "/a-b" },
|
|
167
|
+
});
|
|
168
|
+
const report = writeDecofileToDir({ "pages-A B": { name: "new", path: "/a-b" } }, { out });
|
|
169
|
+
expect(report.updated).toEqual(["pages-A B"]);
|
|
170
|
+
expect(ls(out)).toEqual(["pages-A%20B.json"]);
|
|
171
|
+
expect(read(out, "pages-A%20B.json").name).toBe("new");
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("prunes blocks absent upstream, but keeps denied and secret-bearing ones", () => {
|
|
175
|
+
const out = tmpBlocksDir({
|
|
176
|
+
"pages-Gone.json": { name: "gone", path: "/gone" },
|
|
177
|
+
"pages-Home.json": { name: "home", path: "/" },
|
|
178
|
+
"Site.json": { seo: {} },
|
|
179
|
+
"deco-vtex.json": { appKey: { name: "KEY", encrypted: "local" } },
|
|
180
|
+
});
|
|
181
|
+
const report = writeDecofileToDir(
|
|
182
|
+
{ "pages-Home": { name: "home", path: "/" } },
|
|
183
|
+
{
|
|
184
|
+
out,
|
|
185
|
+
prune: true,
|
|
186
|
+
},
|
|
187
|
+
);
|
|
188
|
+
expect(report.removed).toEqual(["pages-Gone"]);
|
|
189
|
+
expect(report.denied).toEqual(["Site"]);
|
|
190
|
+
expect(report.protectedSecretBlocks).toEqual(["deco-vtex"]);
|
|
191
|
+
expect(ls(out)).toEqual(["Site.json", "deco-vtex.json", "pages-Home.json"]);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
it("--dry-run touches nothing", () => {
|
|
195
|
+
const out = tmpBlocksDir({ "pages-Gone.json": { path: "/gone" } });
|
|
196
|
+
const report = writeDecofileToDir(
|
|
197
|
+
{ "pages-Home": { path: "/" } },
|
|
198
|
+
{
|
|
199
|
+
out,
|
|
200
|
+
prune: true,
|
|
201
|
+
dryRun: true,
|
|
202
|
+
},
|
|
203
|
+
);
|
|
204
|
+
expect(report.added).toEqual(["pages-Home"]);
|
|
205
|
+
expect(report.removed).toEqual(["pages-Gone"]);
|
|
206
|
+
expect(ls(out)).toEqual(["pages-Gone.json"]);
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
it("skips non-object payload values and reports plaintext secrets", () => {
|
|
210
|
+
const out = tmpBlocksDir();
|
|
211
|
+
const report = writeDecofileToDir(
|
|
212
|
+
{ broken: "not a block", leak: { appToken: "abcdefghijklmnop" } },
|
|
213
|
+
{ out },
|
|
214
|
+
);
|
|
215
|
+
expect(report.skipped).toEqual(["broken"]);
|
|
216
|
+
expect(report.plaintextSecrets).toEqual(["leak.appToken"]);
|
|
217
|
+
});
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
describe("fetchDecofile", () => {
|
|
221
|
+
afterEach(() => vi.unstubAllGlobals());
|
|
222
|
+
|
|
223
|
+
const stub = (body: string, init: { status?: number; headers?: Record<string, string> } = {}) => {
|
|
224
|
+
vi.stubGlobal(
|
|
225
|
+
"fetch",
|
|
226
|
+
vi.fn(
|
|
227
|
+
async () =>
|
|
228
|
+
new Response(body, {
|
|
229
|
+
status: init.status ?? 200,
|
|
230
|
+
headers: { "content-type": "application/json", ...init.headers },
|
|
231
|
+
}),
|
|
232
|
+
),
|
|
233
|
+
);
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
it("returns the parsed decofile plus the ETag revision", async () => {
|
|
237
|
+
stub(JSON.stringify({ "pages-Home": { path: "/" } }), { headers: { etag: '"abc123"' } });
|
|
238
|
+
const result = await fetchDecofile("https://x.com/.decofile");
|
|
239
|
+
expect(result.blocks).toEqual({ "pages-Home": { path: "/" } });
|
|
240
|
+
expect(result.revision).toBe('"abc123"');
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
it("rejects a non-200, a non-JSON content-type, a non-object payload and an oversized body", async () => {
|
|
244
|
+
stub("{}", { status: 500 });
|
|
245
|
+
await expect(fetchDecofile("https://x.com/.decofile")).rejects.toThrow(/responded 500/);
|
|
246
|
+
|
|
247
|
+
stub("<html>", { headers: { "content-type": "text/html" } });
|
|
248
|
+
await expect(fetchDecofile("https://x.com/.decofile")).rejects.toThrow(/expected JSON/);
|
|
249
|
+
|
|
250
|
+
stub("[]");
|
|
251
|
+
await expect(fetchDecofile("https://x.com/.decofile")).rejects.toThrow(/an array/);
|
|
252
|
+
|
|
253
|
+
stub(JSON.stringify({ a: { b: 1 } }));
|
|
254
|
+
await expect(fetchDecofile("https://x.com/.decofile", { maxBytes: 4 })).rejects.toThrow(
|
|
255
|
+
/over the 4 byte cap/,
|
|
256
|
+
);
|
|
257
|
+
});
|
|
258
|
+
});
|
|
@@ -0,0 +1,557 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* @decocms/blocks-cli — pull the production decofile into `.deco/blocks/`
|
|
4
|
+
*
|
|
5
|
+
* Secure replacement for the legacy push-based content sync, where a workflow
|
|
6
|
+
* in the *legacy* Fresh repo held a cross-repo PAT and pushed straight into the
|
|
7
|
+
* migrated repo's `main` (`rsync --delete` + `git push`). That token is a code
|
|
8
|
+
* -write door into the new repo, not a content channel.
|
|
9
|
+
*
|
|
10
|
+
* This inverts the direction: the migrated repo *pulls* the decofile from the
|
|
11
|
+
* live site (`GET <origin>/.decofile`, public and unauthenticated) on a daily
|
|
12
|
+
* cron, materialises one file per block, and opens a PR. No cross-repo token,
|
|
13
|
+
* no write permission handed to anyone, and the content passes a build gate
|
|
14
|
+
* before reaching `main`. See `docs/content-sync.md`.
|
|
15
|
+
*
|
|
16
|
+
* Three filters decide what may be overwritten:
|
|
17
|
+
* 1. `--deny <globs>` — deny by block key (default: the `Site` block).
|
|
18
|
+
* 2. encrypted-secret shape — any block carrying a `{name, encrypted}` secret
|
|
19
|
+
* ref anywhere in its tree is left alone. That is what protects the
|
|
20
|
+
* credentials the migration moves onto the new site's own app block
|
|
21
|
+
* (e.g. `deco-vtex`), which do not exist in that layout upstream.
|
|
22
|
+
* Opt out with `--allow-secret-blocks`.
|
|
23
|
+
* 3. `--fail-on-plaintext-secret` — aborts if an accepted block carries what
|
|
24
|
+
* looks like a *plaintext* credential, so a leak upstream is never
|
|
25
|
+
* committed into git.
|
|
26
|
+
*
|
|
27
|
+
* Usage (from a site root):
|
|
28
|
+
* tsx pull-decofile.ts --origin https://www.minhaloja.com.br --prune
|
|
29
|
+
* tsx pull-decofile.ts --url https://www.minhaloja.com.br/.decofile --dry-run --json
|
|
30
|
+
*
|
|
31
|
+
* Exit codes:
|
|
32
|
+
* 0 — done (with or without changes)
|
|
33
|
+
* 1 — `--fail-on-plaintext-secret` and at least one plaintext finding
|
|
34
|
+
* 2 — usage / network / payload validation error (nothing was written)
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
import * as fs from "node:fs";
|
|
38
|
+
import * as path from "node:path";
|
|
39
|
+
import { decodeBlockNameWithPasses } from "./lib/blocks-dedupe";
|
|
40
|
+
|
|
41
|
+
/** Block keys never overwritten by a sync unless `--deny` is overridden. */
|
|
42
|
+
export const DEFAULT_DENY = ["Site", "site"];
|
|
43
|
+
|
|
44
|
+
const DEFAULT_MAX_BYTES = 64 * 1024 * 1024;
|
|
45
|
+
const DEFAULT_TIMEOUT_MS = 60_000;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Property names whose *string* value would be a credential in the clear.
|
|
49
|
+
* A bare `key` is deliberately NOT here: CMS content is full of `key` props
|
|
50
|
+
* that are nothing of the sort (`selectedFacets[].key` on every VTEX PLP
|
|
51
|
+
* loader — 331 false positives on a real site), and a gate that cries wolf is a
|
|
52
|
+
* gate nobody keeps on.
|
|
53
|
+
* `key` only counts when qualified (`apiKey`, `appKey`, `privateKey`, …).
|
|
54
|
+
*/
|
|
55
|
+
const SECRET_PROP_RE =
|
|
56
|
+
/(?:^|_)(?:(?:api|app|access|private|public|client|secret|auth)_?keys?|tokens?|secrets?|passwords?|passwd|pwd)$/;
|
|
57
|
+
|
|
58
|
+
/** `appToken` -> `app_token`, so one snake_case regex covers both styles. */
|
|
59
|
+
function normalizeProp(prop: string): string {
|
|
60
|
+
return prop
|
|
61
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1_$2")
|
|
62
|
+
.replace(/-/g, "_")
|
|
63
|
+
.toLowerCase();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface PullOptions {
|
|
67
|
+
/** Directory that holds one JSON file per block (usually `.deco/blocks`). */
|
|
68
|
+
out: string;
|
|
69
|
+
/** Glob patterns (`*` wildcard) matched against the block key. */
|
|
70
|
+
deny?: string[];
|
|
71
|
+
/** Overwrite blocks that carry an encrypted secret ref (default: false). */
|
|
72
|
+
allowSecretBlocks?: boolean;
|
|
73
|
+
/** Delete local blocks that no longer exist upstream (default: false). */
|
|
74
|
+
prune?: boolean;
|
|
75
|
+
/** Compute the report without touching the filesystem. */
|
|
76
|
+
dryRun?: boolean;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface PullReport {
|
|
80
|
+
added: string[];
|
|
81
|
+
updated: string[];
|
|
82
|
+
unchanged: number;
|
|
83
|
+
removed: string[];
|
|
84
|
+
/** Keys skipped by the deny glob. */
|
|
85
|
+
denied: string[];
|
|
86
|
+
/** Keys skipped because the local/remote block carries an encrypted secret. */
|
|
87
|
+
protectedSecretBlocks: string[];
|
|
88
|
+
/** Keys skipped because the payload value was not a JSON object. */
|
|
89
|
+
skipped: string[];
|
|
90
|
+
/** `<key>.<prop path>` of every plaintext credential found in a written block. */
|
|
91
|
+
plaintextSecrets: string[];
|
|
92
|
+
/** Blocks present in the remote payload. */
|
|
93
|
+
remoteBlocks: number;
|
|
94
|
+
revision?: string;
|
|
95
|
+
bytes?: number;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export interface FetchResult {
|
|
99
|
+
blocks: Record<string, unknown>;
|
|
100
|
+
revision?: string;
|
|
101
|
+
bytes: number;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// ---------------------------------------------------------------- pure helpers
|
|
105
|
+
|
|
106
|
+
/** `*`-only glob match against a whole block key. */
|
|
107
|
+
export function matchesGlob(key: string, pattern: string): boolean {
|
|
108
|
+
const rx = new RegExp(
|
|
109
|
+
`^${pattern.replace(/[.*+?^${}()|[\]\\]/g, (c) => (c === "*" ? "[\\s\\S]*" : `\\${c}`))}$`,
|
|
110
|
+
);
|
|
111
|
+
return rx.test(key);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
|
115
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* True iff the block carries a Deco encrypted-secret ref anywhere in its tree.
|
|
120
|
+
* Prod serves those as `{"name": "MY_TOKEN", "encrypted": "3a714de1c8…"}`.
|
|
121
|
+
*/
|
|
122
|
+
export function hasEncryptedSecretRef(value: unknown): boolean {
|
|
123
|
+
if (Array.isArray(value)) return value.some(hasEncryptedSecretRef);
|
|
124
|
+
if (!isPlainObject(value)) return false;
|
|
125
|
+
if (typeof value.name === "string" && typeof value.encrypted === "string" && value.encrypted) {
|
|
126
|
+
return true;
|
|
127
|
+
}
|
|
128
|
+
return Object.values(value).some(hasEncryptedSecretRef);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Property paths whose name looks credential-shaped and whose value is a raw
|
|
133
|
+
* string — i.e. a secret in the clear rather than an `{name, encrypted}` ref.
|
|
134
|
+
* URLs and values with whitespace are excluded (endpoints, prose, templates).
|
|
135
|
+
*/
|
|
136
|
+
export function findPlaintextSecrets(value: unknown, prefix = ""): string[] {
|
|
137
|
+
const found: string[] = [];
|
|
138
|
+
const walk = (node: unknown, at: string): void => {
|
|
139
|
+
if (Array.isArray(node)) {
|
|
140
|
+
for (const [i, item] of node.entries()) walk(item, `${at}[${i}]`);
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (!isPlainObject(node)) return;
|
|
144
|
+
for (const [prop, child] of Object.entries(node)) {
|
|
145
|
+
const at2 = at ? `${at}.${prop}` : prop;
|
|
146
|
+
if (
|
|
147
|
+
typeof child === "string" &&
|
|
148
|
+
child.length >= 12 &&
|
|
149
|
+
SECRET_PROP_RE.test(normalizeProp(prop)) &&
|
|
150
|
+
!/\s/.test(child) &&
|
|
151
|
+
!/^https?:\/\//i.test(child) &&
|
|
152
|
+
!/^\{\{.*\}\}$/.test(child)
|
|
153
|
+
) {
|
|
154
|
+
found.push(at2);
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
walk(child, at2);
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
walk(value, prefix);
|
|
161
|
+
return found;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Fully URL-decode a block key or filename stem, for cross-scheme matching. */
|
|
165
|
+
function canonicalKey(keyOrFile: string): string {
|
|
166
|
+
return decodeBlockNameWithPasses(keyOrFile).name;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Serialised on-disk form of a block: pretty-printed, so a PR diff shows the
|
|
171
|
+
* sections/props that actually changed instead of one 3 MB line.
|
|
172
|
+
*/
|
|
173
|
+
function serialize(block: unknown): string {
|
|
174
|
+
return `${JSON.stringify(block, null, 2)}\n`;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Key-sorted JSON, for *comparison only*. Writers of `.deco/blocks` disagree on
|
|
179
|
+
* formatting (the Studio daemon and the old bot minify, a hand-made sync PR
|
|
180
|
+
* pretty-prints) and on key order, so a byte compare reports a diff on a block
|
|
181
|
+
* whose content is identical — 124 of 432 blocks on a real site's first run.
|
|
182
|
+
* Compare semantically, write only real content changes.
|
|
183
|
+
*/
|
|
184
|
+
function stableStringify(value: unknown): string {
|
|
185
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
186
|
+
if (!isPlainObject(value)) return JSON.stringify(value) ?? "null";
|
|
187
|
+
const entries = Object.keys(value)
|
|
188
|
+
.sort()
|
|
189
|
+
.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`);
|
|
190
|
+
return `{${entries.join(",")}}`;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// ------------------------------------------------------------------ fetch side
|
|
194
|
+
|
|
195
|
+
/** Download and validate the remote decofile. Throws on any bad payload. */
|
|
196
|
+
export async function fetchDecofile(
|
|
197
|
+
url: string,
|
|
198
|
+
opts: { maxBytes?: number; timeoutMs?: number } = {},
|
|
199
|
+
): Promise<FetchResult> {
|
|
200
|
+
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
201
|
+
const res = await fetch(url, {
|
|
202
|
+
redirect: "follow",
|
|
203
|
+
headers: { accept: "application/json" },
|
|
204
|
+
signal: AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
|
205
|
+
});
|
|
206
|
+
if (!res.ok) throw new Error(`${url} responded ${res.status} ${res.statusText}`);
|
|
207
|
+
|
|
208
|
+
const contentType = res.headers.get("content-type") ?? "";
|
|
209
|
+
if (!contentType.includes("json")) {
|
|
210
|
+
throw new Error(`${url} served "${contentType || "no content-type"}", expected JSON`);
|
|
211
|
+
}
|
|
212
|
+
const declared = Number(res.headers.get("content-length") ?? Number.NaN);
|
|
213
|
+
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
214
|
+
throw new Error(`${url} is ${declared} bytes, over the ${maxBytes} byte cap`);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const body = await res.text();
|
|
218
|
+
if (body.length > maxBytes) {
|
|
219
|
+
throw new Error(`${url} is ${body.length} bytes, over the ${maxBytes} byte cap`);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
let parsed: unknown;
|
|
223
|
+
try {
|
|
224
|
+
parsed = JSON.parse(body);
|
|
225
|
+
} catch (e) {
|
|
226
|
+
throw new Error(`${url} did not return valid JSON: ${(e as Error).message}`);
|
|
227
|
+
}
|
|
228
|
+
if (!isPlainObject(parsed)) {
|
|
229
|
+
throw new Error(
|
|
230
|
+
`${url} returned ${Array.isArray(parsed) ? "an array" : typeof parsed}, expected a decofile object`,
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
return { blocks: parsed, revision: res.headers.get("etag") ?? undefined, bytes: body.length };
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// ------------------------------------------------------------------ write side
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Materialise `remote` into `opts.out`, one file per block.
|
|
241
|
+
*
|
|
242
|
+
* Filenames are `encodeURIComponent(key) + ".json"` — the single-decode scheme
|
|
243
|
+
* the runtime's `parseBlockId` expects. When a file for the same *logical* key
|
|
244
|
+
* already exists under a different encoding (the `deco-sync-bot` wrote
|
|
245
|
+
* double-encoded names), that existing file is overwritten in place instead of
|
|
246
|
+
* a second, colliding one being created.
|
|
247
|
+
*/
|
|
248
|
+
export function writeDecofileToDir(remote: Record<string, unknown>, opts: PullOptions): PullReport {
|
|
249
|
+
const out = path.resolve(opts.out);
|
|
250
|
+
const deny = opts.deny ?? DEFAULT_DENY;
|
|
251
|
+
const report: PullReport = {
|
|
252
|
+
added: [],
|
|
253
|
+
updated: [],
|
|
254
|
+
unchanged: 0,
|
|
255
|
+
removed: [],
|
|
256
|
+
denied: [],
|
|
257
|
+
protectedSecretBlocks: [],
|
|
258
|
+
skipped: [],
|
|
259
|
+
plaintextSecrets: [],
|
|
260
|
+
remoteBlocks: Object.keys(remote).length,
|
|
261
|
+
};
|
|
262
|
+
|
|
263
|
+
fs.mkdirSync(out, { recursive: true });
|
|
264
|
+
|
|
265
|
+
// Index what is already on disk by fully-decoded key, so we overwrite legacy
|
|
266
|
+
// double-encoded filenames instead of duplicating them. A key can map to
|
|
267
|
+
// several files (the `deco-sync-bot` wrote `pages-A%2520B.json` where the
|
|
268
|
+
// manual sync wrote `pages-A%20B.json`).
|
|
269
|
+
const existingByKey = new Map<string, string[]>();
|
|
270
|
+
for (const file of fs.readdirSync(out)) {
|
|
271
|
+
if (!file.endsWith(".json")) continue;
|
|
272
|
+
const key = canonicalKey(file);
|
|
273
|
+
const list = existingByKey.get(key);
|
|
274
|
+
if (list) list.push(file);
|
|
275
|
+
else existingByKey.set(key, [file]);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
const seen = new Set<string>();
|
|
279
|
+
|
|
280
|
+
for (const [key, block] of Object.entries(remote)) {
|
|
281
|
+
const canonical = canonicalKey(key);
|
|
282
|
+
seen.add(canonical);
|
|
283
|
+
|
|
284
|
+
if (deny.some((p) => matchesGlob(key, p) || matchesGlob(canonical, p))) {
|
|
285
|
+
report.denied.push(key);
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
if (!isPlainObject(block)) {
|
|
289
|
+
report.skipped.push(key);
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
if (!opts.allowSecretBlocks && hasEncryptedSecretRef(block)) {
|
|
293
|
+
report.protectedSecretBlocks.push(key);
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// One existing file → write it in place (smallest diff, no rename churn).
|
|
298
|
+
// Several → converge on the canonical single-encoded name and drop the
|
|
299
|
+
// others: leaving a stale duplicate behind is not cosmetic, `pickWinner`
|
|
300
|
+
// in generate-blocks prefers the *more*-encoded filename, so the stale one
|
|
301
|
+
// would win the build.
|
|
302
|
+
const existing = existingByKey.get(canonical) ?? [];
|
|
303
|
+
const canonicalFile = `${encodeURIComponent(key)}.json`;
|
|
304
|
+
const file = existing.length === 1 ? existing[0] : canonicalFile;
|
|
305
|
+
const target = path.join(out, file);
|
|
306
|
+
// encodeURIComponent cannot emit a separator, but assert the boundary
|
|
307
|
+
// anyway: this writes to a git repo from a remote payload.
|
|
308
|
+
if (path.dirname(path.resolve(target)) !== out) {
|
|
309
|
+
throw new Error(`refusing to write block "${key}" outside ${out}`);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const secrets = findPlaintextSecrets(block);
|
|
313
|
+
for (const at of secrets) report.plaintextSecrets.push(`${key}.${at}`);
|
|
314
|
+
|
|
315
|
+
const duplicates = existing.filter((f) => f !== file);
|
|
316
|
+
if (!opts.dryRun) {
|
|
317
|
+
for (const dup of duplicates) fs.rmSync(path.join(out, dup));
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
const next = serialize(block);
|
|
321
|
+
const current = fs.existsSync(target) ? fs.readFileSync(target, "utf-8") : null;
|
|
322
|
+
let currentParsed: unknown;
|
|
323
|
+
try {
|
|
324
|
+
currentParsed = current === null ? undefined : JSON.parse(current);
|
|
325
|
+
} catch {
|
|
326
|
+
currentParsed = undefined; // unparseable local file — overwrite it
|
|
327
|
+
}
|
|
328
|
+
const sameContent =
|
|
329
|
+
current !== null && stableStringify(currentParsed) === stableStringify(block);
|
|
330
|
+
if (sameContent && duplicates.length === 0) {
|
|
331
|
+
report.unchanged++;
|
|
332
|
+
continue;
|
|
333
|
+
}
|
|
334
|
+
if (!opts.dryRun) fs.writeFileSync(target, next);
|
|
335
|
+
(current === null ? report.added : report.updated).push(key);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
if (opts.prune) {
|
|
339
|
+
for (const [key, files] of existingByKey) {
|
|
340
|
+
if (seen.has(key)) continue;
|
|
341
|
+
if (deny.some((p) => matchesGlob(key, p))) {
|
|
342
|
+
report.denied.push(key);
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
// A local-only block holding credentials is site-owned (the migration put
|
|
346
|
+
// them there); upstream never had it, so its absence must not delete it.
|
|
347
|
+
if (!opts.allowSecretBlocks) {
|
|
348
|
+
const local = files.map((f) => {
|
|
349
|
+
try {
|
|
350
|
+
return JSON.parse(fs.readFileSync(path.join(out, f), "utf-8")) as unknown;
|
|
351
|
+
} catch {
|
|
352
|
+
return null;
|
|
353
|
+
}
|
|
354
|
+
});
|
|
355
|
+
if (local.some(hasEncryptedSecretRef)) {
|
|
356
|
+
report.protectedSecretBlocks.push(key);
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
if (!opts.dryRun) {
|
|
361
|
+
for (const f of files) fs.rmSync(path.join(out, f));
|
|
362
|
+
}
|
|
363
|
+
report.removed.push(key);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
return report;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// -------------------------------------------------------------------- CLI
|
|
371
|
+
|
|
372
|
+
interface CliOptions extends PullOptions {
|
|
373
|
+
url?: string;
|
|
374
|
+
maxBytes: number;
|
|
375
|
+
timeoutMs: number;
|
|
376
|
+
failOnPlaintextSecret: boolean;
|
|
377
|
+
json: boolean;
|
|
378
|
+
github: boolean;
|
|
379
|
+
help: boolean;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
function parseArgs(argv: string[]): CliOptions {
|
|
383
|
+
const opts: CliOptions = {
|
|
384
|
+
out: ".deco/blocks",
|
|
385
|
+
maxBytes: DEFAULT_MAX_BYTES,
|
|
386
|
+
timeoutMs: DEFAULT_TIMEOUT_MS,
|
|
387
|
+
failOnPlaintextSecret: false,
|
|
388
|
+
json: false,
|
|
389
|
+
github: false,
|
|
390
|
+
help: false,
|
|
391
|
+
};
|
|
392
|
+
for (let i = 0; i < argv.length; i++) {
|
|
393
|
+
const flag = argv[i];
|
|
394
|
+
switch (flag) {
|
|
395
|
+
case "--url":
|
|
396
|
+
opts.url = argv[++i];
|
|
397
|
+
break;
|
|
398
|
+
case "--origin": {
|
|
399
|
+
const origin = (argv[++i] ?? "").replace(/\/+$/, "");
|
|
400
|
+
opts.url = origin ? `${origin}/.decofile` : undefined;
|
|
401
|
+
break;
|
|
402
|
+
}
|
|
403
|
+
case "--out":
|
|
404
|
+
opts.out = argv[++i] ?? opts.out;
|
|
405
|
+
break;
|
|
406
|
+
case "--deny":
|
|
407
|
+
opts.deny = (argv[++i] ?? "")
|
|
408
|
+
.split(",")
|
|
409
|
+
.map((s) => s.trim())
|
|
410
|
+
.filter(Boolean);
|
|
411
|
+
break;
|
|
412
|
+
case "--allow-secret-blocks":
|
|
413
|
+
opts.allowSecretBlocks = true;
|
|
414
|
+
break;
|
|
415
|
+
case "--prune":
|
|
416
|
+
opts.prune = true;
|
|
417
|
+
break;
|
|
418
|
+
case "--dry-run":
|
|
419
|
+
opts.dryRun = true;
|
|
420
|
+
break;
|
|
421
|
+
case "--fail-on-plaintext-secret":
|
|
422
|
+
opts.failOnPlaintextSecret = true;
|
|
423
|
+
break;
|
|
424
|
+
case "--max-bytes":
|
|
425
|
+
opts.maxBytes = Number(argv[++i]);
|
|
426
|
+
break;
|
|
427
|
+
case "--timeout-ms":
|
|
428
|
+
opts.timeoutMs = Number(argv[++i]);
|
|
429
|
+
break;
|
|
430
|
+
case "--json":
|
|
431
|
+
opts.json = true;
|
|
432
|
+
break;
|
|
433
|
+
case "--github":
|
|
434
|
+
opts.github = true;
|
|
435
|
+
break;
|
|
436
|
+
case "--help":
|
|
437
|
+
case "-h":
|
|
438
|
+
opts.help = true;
|
|
439
|
+
break;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
return opts;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function showHelp(): void {
|
|
446
|
+
console.log(`
|
|
447
|
+
@decocms/blocks-cli — pull the production decofile into .deco/blocks/
|
|
448
|
+
|
|
449
|
+
Usage:
|
|
450
|
+
tsx pull-decofile.ts --origin https://www.minhaloja.com.br [options]
|
|
451
|
+
|
|
452
|
+
Options:
|
|
453
|
+
--origin <url> Site origin; fetches <origin>/.decofile
|
|
454
|
+
--url <url> Full decofile URL (alternative to --origin)
|
|
455
|
+
--out <dir> Blocks directory (default: .deco/blocks)
|
|
456
|
+
--deny <globs> Comma-separated key globs never overwritten
|
|
457
|
+
(default: ${DEFAULT_DENY.join(",")})
|
|
458
|
+
--allow-secret-blocks Also overwrite blocks holding encrypted secrets
|
|
459
|
+
--prune Delete local blocks absent upstream
|
|
460
|
+
--dry-run Report only, write nothing
|
|
461
|
+
--fail-on-plaintext-secret Exit 1 if a written block holds a raw credential
|
|
462
|
+
--max-bytes <n> Payload cap (default: ${DEFAULT_MAX_BYTES})
|
|
463
|
+
--timeout-ms <n> Fetch timeout (default: ${DEFAULT_TIMEOUT_MS})
|
|
464
|
+
--json Emit the report as JSON
|
|
465
|
+
--github Emit ::notice::/::error:: lines for Actions
|
|
466
|
+
--help, -h This message
|
|
467
|
+
|
|
468
|
+
Exit codes:
|
|
469
|
+
0 done 1 plaintext secret gate 2 usage/network error
|
|
470
|
+
`);
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
function reportToText(report: PullReport): string {
|
|
474
|
+
const lines = [
|
|
475
|
+
`remote blocks: ${report.remoteBlocks}${report.bytes ? ` (${report.bytes} bytes)` : ""}${report.revision ? ` revision ${report.revision}` : ""}`,
|
|
476
|
+
`added ${report.added.length} updated ${report.updated.length} unchanged ${report.unchanged} removed ${report.removed.length}`,
|
|
477
|
+
`denied ${report.denied.length} secret-protected ${report.protectedSecretBlocks.length} skipped ${report.skipped.length}`,
|
|
478
|
+
];
|
|
479
|
+
for (const [label, keys] of [
|
|
480
|
+
["added", report.added],
|
|
481
|
+
["updated", report.updated],
|
|
482
|
+
["removed", report.removed],
|
|
483
|
+
] as const) {
|
|
484
|
+
for (const key of keys.slice(0, 50)) lines.push(` ${label}: ${key}`);
|
|
485
|
+
if (keys.length > 50) lines.push(` ${label}: … and ${keys.length - 50} more`);
|
|
486
|
+
}
|
|
487
|
+
return lines.join("\n");
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
async function main(): Promise<void> {
|
|
491
|
+
const opts = parseArgs(process.argv.slice(2));
|
|
492
|
+
if (opts.help) {
|
|
493
|
+
showHelp();
|
|
494
|
+
process.exit(0);
|
|
495
|
+
}
|
|
496
|
+
if (!opts.url) {
|
|
497
|
+
console.error("pull-decofile: --origin or --url is required (see --help)");
|
|
498
|
+
process.exit(2);
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
let fetched: FetchResult;
|
|
502
|
+
try {
|
|
503
|
+
fetched = await fetchDecofile(opts.url, { maxBytes: opts.maxBytes, timeoutMs: opts.timeoutMs });
|
|
504
|
+
} catch (e) {
|
|
505
|
+
console.error(`pull-decofile: ${(e as Error).message}`);
|
|
506
|
+
process.exit(2);
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
let report: PullReport;
|
|
510
|
+
try {
|
|
511
|
+
report = writeDecofileToDir(fetched.blocks, opts);
|
|
512
|
+
} catch (e) {
|
|
513
|
+
console.error(`pull-decofile: ${(e as Error).message}`);
|
|
514
|
+
process.exit(2);
|
|
515
|
+
}
|
|
516
|
+
report.revision = fetched.revision;
|
|
517
|
+
report.bytes = fetched.bytes;
|
|
518
|
+
|
|
519
|
+
process.stdout.write(
|
|
520
|
+
`${opts.json ? JSON.stringify({ url: opts.url, ...report }, null, 2) : reportToText(report)}\n`,
|
|
521
|
+
);
|
|
522
|
+
|
|
523
|
+
if (opts.github) {
|
|
524
|
+
process.stdout.write(
|
|
525
|
+
`::notice::decofile sync — +${report.added.length} ~${report.updated.length} -${report.removed.length} (${report.remoteBlocks} blocks upstream)\n`,
|
|
526
|
+
);
|
|
527
|
+
for (const at of report.plaintextSecrets) {
|
|
528
|
+
process.stdout.write(
|
|
529
|
+
`::error title=plaintext-secret::${at} looks like a credential in the clear\n`,
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
if (report.plaintextSecrets.length > 0 && opts.failOnPlaintextSecret) {
|
|
535
|
+
console.error(
|
|
536
|
+
`pull-decofile: ${report.plaintextSecrets.length} plaintext credential(s) in the pulled content — refusing to commit it. Move them to encrypted secrets upstream, or deny the block with --deny.`,
|
|
537
|
+
);
|
|
538
|
+
process.exit(1);
|
|
539
|
+
}
|
|
540
|
+
process.exit(0);
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
const isCjsEntry =
|
|
544
|
+
typeof require !== "undefined" && typeof module !== "undefined" && require.main === module;
|
|
545
|
+
let isEsmEntry = false;
|
|
546
|
+
try {
|
|
547
|
+
isEsmEntry =
|
|
548
|
+
typeof process !== "undefined" &&
|
|
549
|
+
Array.isArray(process.argv) &&
|
|
550
|
+
process.argv[1] !== undefined &&
|
|
551
|
+
import.meta.url === `file://${process.argv[1]}`;
|
|
552
|
+
} catch {
|
|
553
|
+
// ignore in CJS
|
|
554
|
+
}
|
|
555
|
+
if (isCjsEntry || isEsmEntry) {
|
|
556
|
+
void main();
|
|
557
|
+
}
|