@x12i/catalox 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -6
- package/dist/src/catalox/authorization.js +1 -1
- package/dist/src/catalox/authorization.js.map +1 -1
- package/dist/src/catalox/backup-data.d.ts +0 -6
- package/dist/src/catalox/backup-data.d.ts.map +1 -1
- package/dist/src/catalox/backup-data.js +2 -413
- package/dist/src/catalox/backup-data.js.map +1 -1
- package/dist/src/catalox/catalog-lifecycle.d.ts +29 -0
- package/dist/src/catalox/catalog-lifecycle.d.ts.map +1 -0
- package/dist/src/catalox/catalog-lifecycle.js +480 -0
- package/dist/src/catalox/catalog-lifecycle.js.map +1 -0
- package/dist/src/catalox/catalox-bound.d.ts +21 -6
- package/dist/src/catalox/catalox-bound.d.ts.map +1 -1
- package/dist/src/catalox/catalox-bound.js +30 -9
- package/dist/src/catalox/catalox-bound.js.map +1 -1
- package/dist/src/catalox/catalox.d.ts +31 -11
- package/dist/src/catalox/catalox.d.ts.map +1 -1
- package/dist/src/catalox/catalox.js +485 -82
- package/dist/src/catalox/catalox.js.map +1 -1
- package/dist/src/catalox/context.js +2 -2
- package/dist/src/catalox/context.js.map +1 -1
- package/dist/src/catalox/create-catalox.d.ts +6 -0
- package/dist/src/catalox/create-catalox.d.ts.map +1 -1
- package/dist/src/catalox/create-catalox.js +25 -0
- package/dist/src/catalox/create-catalox.js.map +1 -1
- package/dist/src/catalox/index.d.ts +4 -2
- package/dist/src/catalox/index.d.ts.map +1 -1
- package/dist/src/catalox/index.js +4 -2
- package/dist/src/catalox/index.js.map +1 -1
- package/dist/src/catalox/native-catalog-merge.d.ts +12 -0
- package/dist/src/catalox/native-catalog-merge.d.ts.map +1 -0
- package/dist/src/catalox/native-catalog-merge.js +102 -0
- package/dist/src/catalox/native-catalog-merge.js.map +1 -0
- package/dist/src/catalox/native-scope.d.ts +28 -0
- package/dist/src/catalox/native-scope.d.ts.map +1 -0
- package/dist/src/catalox/native-scope.js +184 -0
- package/dist/src/catalox/native-scope.js.map +1 -0
- package/dist/src/catalox/record-history.d.ts +53 -0
- package/dist/src/catalox/record-history.d.ts.map +1 -0
- package/dist/src/catalox/record-history.js +158 -0
- package/dist/src/catalox/record-history.js.map +1 -0
- package/dist/src/catalox/restore-firestore-backup.d.ts +6 -3
- package/dist/src/catalox/restore-firestore-backup.d.ts.map +1 -1
- package/dist/src/catalox/restore-firestore-backup.js +3 -224
- package/dist/src/catalox/restore-firestore-backup.js.map +1 -1
- package/dist/src/cli/index.js +159 -55
- package/dist/src/cli/index.js.map +1 -1
- package/dist/src/contracts/apps.d.ts +2 -0
- package/dist/src/contracts/apps.d.ts.map +1 -1
- package/dist/src/contracts/backup.d.ts +2 -2
- package/dist/src/contracts/backup.d.ts.map +1 -1
- package/dist/src/contracts/catalog-lifecycle.d.ts +70 -0
- package/dist/src/contracts/catalog-lifecycle.d.ts.map +1 -0
- package/dist/src/contracts/catalog-lifecycle.js +2 -0
- package/dist/src/contracts/catalog-lifecycle.js.map +1 -0
- package/dist/src/contracts/catalogs.d.ts +37 -0
- package/dist/src/contracts/catalogs.d.ts.map +1 -1
- package/dist/src/contracts/catalogs.js.map +1 -1
- package/dist/src/contracts/context.d.ts +5 -1
- package/dist/src/contracts/context.d.ts.map +1 -1
- package/dist/src/contracts/descriptors.d.ts +6 -0
- package/dist/src/contracts/descriptors.d.ts.map +1 -1
- package/dist/src/contracts/index.d.ts +6 -3
- package/dist/src/contracts/index.d.ts.map +1 -1
- package/dist/src/contracts/index.js.map +1 -1
- package/dist/src/contracts/items.d.ts +19 -0
- package/dist/src/contracts/items.d.ts.map +1 -1
- package/dist/src/contracts/record-history.d.ts +66 -0
- package/dist/src/contracts/record-history.d.ts.map +1 -0
- package/dist/src/contracts/record-history.js +2 -0
- package/dist/src/contracts/record-history.js.map +1 -0
- package/dist/src/contracts/restore.d.ts +0 -39
- package/dist/src/contracts/restore.d.ts.map +1 -1
- package/dist/src/firebase/adapter-store.d.ts +1 -0
- package/dist/src/firebase/adapter-store.d.ts.map +1 -1
- package/dist/src/firebase/adapter-store.js +3 -0
- package/dist/src/firebase/adapter-store.js.map +1 -1
- package/dist/src/firebase/binding-store.d.ts +2 -0
- package/dist/src/firebase/binding-store.d.ts.map +1 -1
- package/dist/src/firebase/binding-store.js +10 -0
- package/dist/src/firebase/binding-store.js.map +1 -1
- package/dist/src/firebase/catalog-data-index-store.d.ts +1 -0
- package/dist/src/firebase/catalog-data-index-store.d.ts.map +1 -1
- package/dist/src/firebase/catalog-data-index-store.js +3 -0
- package/dist/src/firebase/catalog-data-index-store.js.map +1 -1
- package/dist/src/firebase/catalog-item-history-store.d.ts +21 -0
- package/dist/src/firebase/catalog-item-history-store.d.ts.map +1 -0
- package/dist/src/firebase/catalog-item-history-store.js +61 -0
- package/dist/src/firebase/catalog-item-history-store.js.map +1 -0
- package/dist/src/firebase/catalog-store.d.ts +1 -0
- package/dist/src/firebase/catalog-store.d.ts.map +1 -1
- package/dist/src/firebase/catalog-store.js +3 -0
- package/dist/src/firebase/catalog-store.js.map +1 -1
- package/dist/src/firebase/index.d.ts +1 -0
- package/dist/src/firebase/index.d.ts.map +1 -1
- package/dist/src/firebase/index.js +1 -0
- package/dist/src/firebase/index.js.map +1 -1
- package/dist/src/firebase/mapping-store.d.ts +1 -0
- package/dist/src/firebase/mapping-store.d.ts.map +1 -1
- package/dist/src/firebase/mapping-store.js +3 -0
- package/dist/src/firebase/mapping-store.js.map +1 -1
- package/dist/src/firebase/native-item-store.d.ts +8 -2
- package/dist/src/firebase/native-item-store.d.ts.map +1 -1
- package/dist/src/firebase/native-item-store.js +22 -6
- package/dist/src/firebase/native-item-store.js.map +1 -1
- package/dist/src/firebase/reference-store.d.ts +3 -0
- package/dist/src/firebase/reference-store.d.ts.map +1 -1
- package/dist/src/firebase/reference-store.js +16 -0
- package/dist/src/firebase/reference-store.js.map +1 -1
- package/dist/src/firebase/renderer-snippet-store.d.ts +3 -0
- package/dist/src/firebase/renderer-snippet-store.d.ts.map +1 -1
- package/dist/src/firebase/renderer-snippet-store.js +17 -0
- package/dist/src/firebase/renderer-snippet-store.js.map +1 -1
- package/dist/src/firebase/snapshot-store.d.ts +1 -0
- package/dist/src/firebase/snapshot-store.d.ts.map +1 -1
- package/dist/src/firebase/snapshot-store.js +8 -0
- package/dist/src/firebase/snapshot-store.js.map +1 -1
- package/dist/test/integration/backup-data-gcs.live.test.d.ts +2 -0
- package/dist/test/integration/backup-data-gcs.live.test.d.ts.map +1 -0
- package/dist/test/integration/backup-data-gcs.live.test.js +98 -0
- package/dist/test/integration/backup-data-gcs.live.test.js.map +1 -0
- package/dist/test/integration/firestore.emulator.test.js +25 -4
- package/dist/test/integration/firestore.emulator.test.js.map +1 -1
- package/dist/test/integration/record-history.live.test.d.ts +2 -0
- package/dist/test/integration/record-history.live.test.d.ts.map +1 -0
- package/dist/test/integration/record-history.live.test.js +141 -0
- package/dist/test/integration/record-history.live.test.js.map +1 -0
- package/dist/test/unit/native-catalog-merge.test.d.ts +2 -0
- package/dist/test/unit/native-catalog-merge.test.d.ts.map +1 -0
- package/dist/test/unit/native-catalog-merge.test.js +33 -0
- package/dist/test/unit/native-catalog-merge.test.js.map +1 -0
- package/dist/test/unit/native-scope.test.d.ts +2 -0
- package/dist/test/unit/native-scope.test.d.ts.map +1 -0
- package/dist/test/unit/native-scope.test.js +29 -0
- package/dist/test/unit/native-scope.test.js.map +1 -0
- package/dist/test/unit/record-history-path.test.d.ts +2 -0
- package/dist/test/unit/record-history-path.test.d.ts.map +1 -0
- package/dist/test/unit/record-history-path.test.js +24 -0
- package/dist/test/unit/record-history-path.test.js.map +1 -0
- package/firestore.indexes.json +39 -0
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -8,6 +8,8 @@ Catalox is a **data-tier** package for managing **app-scoped catalogs** in Fireb
|
|
|
8
8
|
- **Mapped catalogs** (items normalized from MongoDB or APIs)
|
|
9
9
|
- **References + validation contracts** (standardized cross-catalog shapes)
|
|
10
10
|
- **Seed/import/export + batch upsert** workflows for native catalogs
|
|
11
|
+
- **Optional per-record native history** — NDJSON payloads in GCS plus a Firestore `catalogItemHistory` index; list/show/restore/replay APIs and `catalox history …` ([`docs/record-history.md`](docs/record-history.md))
|
|
12
|
+
- **Optional catalog lifecycle** — hard delete (with manifest), restore from manifest, and hard rename across collections; APIs + `catalox catalog …` ([`docs/catalog-crud.md`](docs/catalog-crud.md))
|
|
11
13
|
- **Optional GCS tools** — NDJSON export/import of Firestore collections to a bucket, **compare** live data vs bucket snapshots, **`backupData` mode `gcs`** (Catalox-shaped NDJSON + manifest under a bucket prefix), and matching CLI commands (`firestore export-gcs`, `import-gcs`, `compare-gcs`, `firestore backup --mode gcs`; see [`docs/firestore-gcs-export.md`](docs/firestore-gcs-export.md), [`docs/backup.md`](docs/backup.md))
|
|
12
14
|
|
|
13
15
|
Catalox **does not** own UI, workflow orchestration, remote execution, artifact blobs, or secret storage.
|
|
@@ -52,7 +54,9 @@ Scoping:
|
|
|
52
54
|
- You can scope the service account to **Firestore-related IAM roles** and to a specific **project**.
|
|
53
55
|
- You generally cannot scope an Admin SDK credential to only certain collections/documents via IAM the way client rules work.
|
|
54
56
|
|
|
55
|
-
To connect to a real Firebase project, initialize `firebase-admin` using `GOOGLE_APPLICATION_CREDENTIALS` (service account JSON) or your preferred Admin initialization strategy.
|
|
57
|
+
To connect to a real Firebase project, initialize `firebase-admin` using `GOOGLE_APPLICATION_CREDENTIALS` (service account JSON), workload identity, or your preferred Admin initialization strategy.
|
|
58
|
+
|
|
59
|
+
This repo’s CLI + live tests typically load `.env` via **`@x12i/env`** and support **`GOOGLE_SERVICE_ACCOUNT_BASE64`** as a convenient local/dev secret carrier. That value is picked up **automatically** by the supporting x12i tooling (including **`@x12i/helpers/gcs`**) when present—**upstream code does not need to plumb it through**; just set it in the environment.
|
|
56
60
|
|
|
57
61
|
Minimal example:
|
|
58
62
|
|
|
@@ -79,10 +83,15 @@ Example `.env` (do not commit secrets):
|
|
|
79
83
|
```bash
|
|
80
84
|
MONGO_URI=mongodb://127.0.0.1:27017
|
|
81
85
|
GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
|
|
86
|
+
# Alternative (supported by this repo’s CLI/tests and x12i helpers): base64-encoded service account JSON
|
|
87
|
+
# GOOGLE_SERVICE_ACCOUNT_BASE64=...
|
|
82
88
|
# Or (used by the packaged CLI migration + live integration tests):
|
|
83
89
|
FIREBASE_SERVICE_ACCOUNT_PATH=/path/to/service-account.json
|
|
84
90
|
FIREBASE_PROJECT_ID=your-project-id
|
|
85
91
|
FIRESTORE_LIVE_TESTS=1
|
|
92
|
+
# Optional: record-history live test + CLI (`createCatalox`); use a dedicated test bucket:
|
|
93
|
+
# CATALOX_RECORD_HISTORY_BUCKET=your-test-bucket
|
|
94
|
+
# CATALOX_RECORD_HISTORY_PREFIX=catalox-record-history/
|
|
86
95
|
# Optional defaults for the packaged `catalox` CLI (Catalox app/store context, not GCP):
|
|
87
96
|
# CATALOX_APP_ID=myAppId
|
|
88
97
|
# CATALOX_STORE_ID=myStoreId
|
|
@@ -98,6 +107,9 @@ The `catalox` binary loads `.env` via `dotenv`. Use **`CATALOX_*`** for **Catalo
|
|
|
98
107
|
- **`CATALOX_STORE_ID`** — Default `storeId` for **`report`** and **`export`** when `--store` is omitted (`--store` overrides).
|
|
99
108
|
- **`CATALOX_USER_ID`** — Optional user id / actor for authz-sensitive CLI paths.
|
|
100
109
|
- **`CATALOX_MONGO_URI`** — If set, enables the Mongo catalog adapter in the CLI (wired via **`createCatalox`** in `src/catalox/create-catalox.ts`, called from `src/cli/index.ts`).
|
|
110
|
+
- **`CATALOX_RECORD_HISTORY_BUCKET`** — If set, **`createCatalox`** wires **`recordHistory`** (GCS payloads + **`catalogItemHistory`** on native writes; also required for **`catalox history …`** to persist new events).
|
|
111
|
+
- **`CATALOX_RECORD_HISTORY_PREFIX`** — Optional GCS prefix for record history (default `catalox-record-history/`).
|
|
112
|
+
- **`CATALOX_RECORD_HISTORY_FAIL_CLOSED`** — When **`1`**, failed history writes fail the parent Firestore mutation.
|
|
101
113
|
|
|
102
114
|
**Running commands from this repository:** the shell command `catalox` is only on your `PATH` if the package is installed globally (`npm i -g @x12i/catalox`) or linked (`npm link` from this repo). Otherwise, after `npm run build`, use:
|
|
103
115
|
|
|
@@ -120,6 +132,13 @@ npm run cli -- firestore report-native-layout --app "narrix"
|
|
|
120
132
|
| `firestore export-gcs` / `import-gcs` | NDJSON per collection + optional manifest ([`docs/firestore-gcs-export.md`](docs/firestore-gcs-export.md)). |
|
|
121
133
|
| `firestore compare-gcs` | Diff live Firestore vs bucket NDJSON (single collection or manifest). |
|
|
122
134
|
|
|
135
|
+
#### `history` / `catalog` CLI (3.1+)
|
|
136
|
+
|
|
137
|
+
| Command | Purpose |
|
|
138
|
+
|---------|---------|
|
|
139
|
+
| `history list` / `history show` / `history restore` / `history replay` | Per-record native history from **`catalogItemHistory`** + GCS ([`docs/record-history.md`](docs/record-history.md)). |
|
|
140
|
+
| `catalog delete` / `catalog restore` / `catalog rename` | Hard delete with manifest, restore from manifest, or hard rename ([`docs/catalog-crud.md`](docs/catalog-crud.md)). |
|
|
141
|
+
|
|
123
142
|
## Firestore data model (logical collections)
|
|
124
143
|
|
|
125
144
|
For a **full layout** (subcollections, document id conventions, snapshot runs, and query notes), see [`docs/firestore-data-model.md`](docs/firestore-data-model.md). For **native** catalogs specifically (per-catalog `catalogData-*-items`, layout resolution, `listCatalogItems`, filters, troubleshooting), see [`docs/native-catalog-storage-and-api.md`](docs/native-catalog-storage-and-api.md).
|
|
@@ -136,6 +155,7 @@ Metadata:
|
|
|
136
155
|
- `catalogDescriptors/{catalogId}` (**descriptor metadata** for generic consumption)
|
|
137
156
|
- `catalogRendererSnippets/{catalogId}:{role}[:{mode}]` (**stored renderer snippets** for list/grid/item/report/dashboard rendering)
|
|
138
157
|
- `catalogReferences/{referenceId}` (standardized reference records)
|
|
158
|
+
- `catalogItemHistory/{eventId}` (optional **native write history** index; payloads in GCS — [`docs/record-history.md`](docs/record-history.md))
|
|
139
159
|
|
|
140
160
|
Data:
|
|
141
161
|
|
|
@@ -162,7 +182,11 @@ import { createCatalox } from "@x12i/catalox";
|
|
|
162
182
|
import { getFirestore } from "firebase-admin/firestore";
|
|
163
183
|
|
|
164
184
|
const firestore = getFirestore();
|
|
165
|
-
const catalox = createCatalox({
|
|
185
|
+
const catalox = createCatalox({
|
|
186
|
+
firestore,
|
|
187
|
+
// Optional: per-record native history to GCS + `catalogItemHistory` (or set CATALOX_RECORD_HISTORY_BUCKET for CLI)
|
|
188
|
+
// recordHistory: { gcsBucket: "my-bucket", gcsPrefix: "catalox-record-history/" },
|
|
189
|
+
});
|
|
166
190
|
|
|
167
191
|
// Optional: bind app/tenant context once (no globals; same semantics as passing context each call).
|
|
168
192
|
const scoped = catalox.withContext({ appId: "myApp" });
|
|
@@ -355,7 +379,7 @@ Catalox includes provisioning helpers to reduce “manual Firestore wiring”.
|
|
|
355
379
|
### Create a native catalog (also seeds a minimal descriptor)
|
|
356
380
|
|
|
357
381
|
```ts
|
|
358
|
-
const ctx = { appId: "myAppId",
|
|
382
|
+
const ctx = { appId: "myAppId", superAdmin: true }; // set only after host auth; often from AppRecord.superAdminApp
|
|
359
383
|
|
|
360
384
|
await catalox.createCatalog(ctx, {
|
|
361
385
|
catalogId: "signals",
|
|
@@ -485,7 +509,7 @@ Validation APIs exist with standardized contracts, but **current behavior is min
|
|
|
485
509
|
|
|
486
510
|
## Publishing
|
|
487
511
|
|
|
488
|
-
From a clean tree, **`npm publish`** runs **`prepublishOnly`**, which executes **`npm run build`** then **`npm test`** (unit tests). Ensure `dist/` is build output only; the published tarball includes **`dist`**, **`README.md`**, and **`
|
|
512
|
+
From a clean tree, **`npm publish`** runs **`prepublishOnly`**, which executes **`npm run build`** then **`npm test`** (unit tests). Ensure `dist/` is build output only; the published tarball includes **`dist`**, **`README.md`**, **`LICENSE`**, and **`firestore.indexes.json`** (composite indexes for `catalogItemHistory` queries) per `package.json` **`files`**.
|
|
489
513
|
|
|
490
514
|
## Tests
|
|
491
515
|
|
|
@@ -507,16 +531,29 @@ Integration tests are **live-only** (no mocks, no emulators). They require:
|
|
|
507
531
|
- `FIREBASE_SERVICE_ACCOUNT_PATH=...`
|
|
508
532
|
- `FIREBASE_PROJECT_ID=...`
|
|
509
533
|
|
|
510
|
-
The **`Firestore live integration`** test (`test/integration/firestore.emulator.test.ts`) uses **`createCatalox`** and asserts v3 contracts: **`listCatalogItems`** returns **`listOutcome: "ok"`**, and **`getCatalogItem`** returns **`{ outcome: "found", item }`** for the seeded row.
|
|
534
|
+
The **`Firestore live integration`** test (`test/integration/firestore.emulator.test.ts`) uses **`createCatalox`** and asserts v3 contracts: **`listCatalogItems`** returns **`listOutcome: "ok"`**, and **`getCatalogItem`** returns **`{ outcome: "found", item }`** for the seeded row. The descriptor patch in that test includes **`queryableFields` with `indexed: true`** for filtered fields so **`deriveIndexed`** persists **`indexed.*`** (see **Canonical `indexed` rule** above).
|
|
535
|
+
|
|
536
|
+
When **`CATALOX_RECORD_HISTORY_BUCKET`** is set, **`test/integration/record-history.live.test.ts`** runs as well: ephemeral app/catalog, **`upsert` → `update` → `listCatalogItemHistory` → `getCatalogItemHistoryEvent` → `restoreCatalogItemFromHistory`**, then best-effort Firestore + GCS cleanup for the created history objects.
|
|
511
537
|
|
|
512
538
|
### Live test safety (read before running)
|
|
513
539
|
|
|
514
540
|
- **Never run against production credentials/projects.** Use a dedicated Firebase project for tests.
|
|
515
|
-
- **Touched collections**: `apps`, `catalogs`, `catalogBindings`, `catalogDefinitions`, `catalogDescriptors`, `catalogData/{catalogId}`,
|
|
541
|
+
- **Touched collections**: `apps`, `catalogs`, `catalogBindings`, `catalogDefinitions`, `catalogDescriptors`, `catalogData/{catalogId}`, `catalogData-{catalogId}-items/...`, and (when record history is enabled) `catalogItemHistory/...`. The **`catalox firestore restore-backup`** / **`undo-restore-backup`** commands additionally write `backup-restoreSessions` and `{restoreSessionId}__preRestore-*` sidecar collections (see [`docs/restore-firestore-backup.md`](docs/restore-firestore-backup.md)).
|
|
516
542
|
- **Cleanup**: tests do best-effort deletes of the docs they created, but do not guarantee full teardown.
|
|
517
543
|
|
|
518
544
|
## Changelog
|
|
519
545
|
|
|
546
|
+
### 3.1.1
|
|
547
|
+
|
|
548
|
+
- **Live tests:** Firestore integration test now sets **`queryableFields`** on the patched descriptor so equality filters match stored **`indexed.*`** rows (same rule as production descriptors).
|
|
549
|
+
- **Live tests:** Record-history integration test covers **update**, **get event**, **restore**, and **GCS object** cleanup when a bucket is configured.
|
|
550
|
+
- **Package:** publish **`firestore.indexes.json`** in the npm tarball for operators deploying **`catalogItemHistory`** queries.
|
|
551
|
+
|
|
552
|
+
### 3.1.0
|
|
553
|
+
|
|
554
|
+
- **Per-record history:** optional **`recordHistory`** on **`createCatalox`** (or **`CATALOX_RECORD_HISTORY_*`** env for CLI) writes **GCS NDJSON** + Firestore **`catalogItemHistory`** on native upsert/update/delete/batch/move. APIs: **`listCatalogItemHistory`**, **`getCatalogItemHistoryEvent`**, **`restoreCatalogItemFromHistory`**, **`replayCatalogToPointInTime`**. CLI: **`catalox history …`**. See [`docs/record-history.md`](docs/record-history.md), [`firestore.indexes.json`](firestore.indexes.json).
|
|
555
|
+
- **Catalog lifecycle:** **`deleteCatalog`**, **`restoreDeletedCatalog`**, **`renameCatalog`** (hard rename) + CLI **`catalox catalog …`**. See [`docs/catalog-crud.md`](docs/catalog-crud.md).
|
|
556
|
+
|
|
520
557
|
### 3.0.0
|
|
521
558
|
|
|
522
559
|
- **`createCatalox(config)`** — single factory for Firestore-backed stores, authz, optional Mongo/API adapters, optional renderer snippet store (`src/catalox/create-catalox.ts`).
|
|
@@ -6,7 +6,7 @@ export class AuthorizationService {
|
|
|
6
6
|
this.bindings = bindings;
|
|
7
7
|
}
|
|
8
8
|
async requireBindingAccess(context, appId, catalogId, required) {
|
|
9
|
-
if (context.
|
|
9
|
+
if (context.superAdmin) {
|
|
10
10
|
// God-mode policy: bypass binding for read/admin checks if needed.
|
|
11
11
|
// We still return a synthetic binding for uniformity.
|
|
12
12
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"authorization.js","sourceRoot":"","sources":["../../../src/catalox/authorization.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAI5D,MAAM,OAAO,oBAAoB;IACF;IAA7B,YAA6B,QAAsB;QAAtB,aAAQ,GAAR,QAAQ,CAAc;IAAG,CAAC;IAEvD,KAAK,CAAC,oBAAoB,CACxB,OAAuB,EACvB,KAAY,EACZ,SAAoB,EACpB,QAAwB;QAExB,IAAI,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"authorization.js","sourceRoot":"","sources":["../../../src/catalox/authorization.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAI5D,MAAM,OAAO,oBAAoB;IACF;IAA7B,YAA6B,QAAsB;QAAtB,aAAQ,GAAR,QAAQ,CAAc;IAAG,CAAC;IAEvD,KAAK,CAAC,oBAAoB,CACxB,OAAuB,EACvB,KAAY,EACZ,SAAoB,EACpB,QAAwB;QAExB,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,mEAAmE;YACnE,sDAAsD;YACtD,OAAO;gBACL,SAAS,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,EAAE;gBAClD,KAAK;gBACL,SAAS;gBACT,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACzD,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;gBACpC,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;aACrC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACvE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC5C,MAAM,IAAI,wBAAwB,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,+BAA+B,EAAE,CAAC,CAAC;QACpG,CAAC;QAED,MAAM,EAAE,GACN,QAAQ,KAAK,MAAM;YACjB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;YACjC,CAAC,CAAC,QAAQ,KAAK,OAAO;gBACpB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAClC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEzC,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,MAAM,IAAI,wBAAwB,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { Readable } from "node:stream";
|
|
2
2
|
import { type Firestore } from "firebase-admin/firestore";
|
|
3
|
-
import { type Db } from "mongodb";
|
|
4
3
|
import type { CatalogId } from "../contracts/ids.js";
|
|
5
4
|
import type { BackupDataInput, BackupDataResult } from "../contracts/backup.js";
|
|
6
5
|
import type { PruneGcsBackupRunsInput, PruneGcsBackupRunsResult } from "../contracts/restore.js";
|
|
@@ -29,8 +28,6 @@ export declare function discoverNativeCatalogIds(fs: Firestore, catalogs: Catalo
|
|
|
29
28
|
catalogIds?: CatalogId[];
|
|
30
29
|
appId?: string;
|
|
31
30
|
}): Promise<CatalogId[]>;
|
|
32
|
-
/** Delete Mongo `backupRuns` documents with createdAt older than cutoff ISO string. */
|
|
33
|
-
export declare function pruneMongoBackupRuns(db: Db, createdBeforeIso: string): Promise<number>;
|
|
34
31
|
/** Exported for tests: `{gcsPrefix|default}/{formatBackupTimestamp}/` without trailing slash (helpers client prefix). */
|
|
35
32
|
export declare function gcsBackupRunFolder(ts: string, gcsPrefix?: string): string;
|
|
36
33
|
/** Subset of `@x12i/helpers/gcs` client used by Catalox backup / restore / prune. */
|
|
@@ -68,8 +65,5 @@ export declare function deleteCataloxGcsBackupRunObjects(bucket: string, runFold
|
|
|
68
65
|
* Folder names are sorted lexicographically (compatible with `formatBackupTimestamp` ISO-style tokens).
|
|
69
66
|
*/
|
|
70
67
|
export declare function pruneGcsBackupRuns(input: PruneGcsBackupRunsInput): Promise<PruneGcsBackupRunsResult>;
|
|
71
|
-
export declare function pruneFirebaseVersionedBackups(firestore: Firestore, keepLast: number): Promise<{
|
|
72
|
-
deleted: string[];
|
|
73
|
-
}>;
|
|
74
68
|
export declare function runBackupData(deps: BackupDataDeps, input: BackupDataInput): Promise<BackupDataResult>;
|
|
75
69
|
//# sourceMappingURL=backup-data.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backup-data.d.ts","sourceRoot":"","sources":["../../../src/catalox/backup-data.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAGL,KAAK,SAAS,EAEf,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"backup-data.d.ts","sourceRoot":"","sources":["../../../src/catalox/backup-data.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAGL,KAAK,SAAS,EAEf,MAAM,0BAA0B,CAAC;AAElC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAmB,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AACjG,OAAO,KAAK,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACjG,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAUhE,eAAO,MAAM,4BAA4B,4LAY/B,CAAC;AAEX,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,EAAE,YAAY,CAAC;IACvB,QAAQ,EAAE,YAAY,CAAC;CACxB,CAAC;AAEF,wBAAgB,qBAAqB,CAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC,OAAa,GAAG,MAAM,CAMtF;AAMD,2GAA2G;AAC3G,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,+CAA+C;AAC/C,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAEzE;AAID,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAE/D;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjE;AAED,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnF;AAED,wBAAsB,iCAAiC,CACrD,EAAE,EAAE,SAAS,EACb,oBAAoB,EAAE,MAAM,EAC5B,kBAAkB,EAAE,MAAM,EAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC9B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED,wBAAsB,kCAAkC,CAAC,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAS7G;AAED,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,aAAa,EAAE,EACzB,IAAI,EAAE;IAAE,QAAQ,EAAE,YAAY,CAAC;IAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACzE,OAAO,CAAC,SAAS,EAAE,CAAC,CAwBtB;AA4BD,yHAAyH;AACzH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAIzE;AAED,qFAAqF;AACrF,MAAM,MAAM,uBAAuB,GAAG;IACpC,YAAY,EAAE,CACZ,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,QAAQ,EAC7C,IAAI,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,KACjD,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,cAAc,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,QAAQ,CAAC;IACjF,gBAAgB,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1F,WAAW,EAAE,CAAC,KAAK,CAAC,EAAE;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,KAAK,OAAO,CAAC;QAAE,KAAK,CAAC,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5F,YAAY,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,YAAY,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACrD,CAAC;AAEF,eAAO,MAAM,oCAAoC,iCAAiC,CAAC;AAYnF,uGAAuG;AACvG,wBAAgB,6BAA6B,CAAC,MAAM,EAAE,MAAM,EAAE,0BAA0B,EAAE,MAAM,GAAG,uBAAuB,CAEzH;AAED,2EAA2E;AAC3E,wBAAsB,gCAAgC,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAwBtH;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,uBAAuB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAmD1G;AA8LD,wBAAsB,aAAa,CACjC,IAAI,EAAE,cAAc,EACpB,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,gBAAgB,CAAC,CAkB3B"}
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import { createRequire } from "node:module";
|
|
2
2
|
import { Readable } from "node:stream";
|
|
3
3
|
import { FieldPath, } from "firebase-admin/firestore";
|
|
4
|
-
import { MongoClient } from "mongodb";
|
|
5
4
|
import { CatalogStore } from "../firebase/catalog-store.js";
|
|
6
5
|
import { BindingStore } from "../firebase/binding-store.js";
|
|
7
6
|
import { FirestoreStore } from "../firebase/firestore-store.js";
|
|
8
7
|
import { flatNativeItemsCollectionRef, legacyNativeItemsCollectionRef, nativeItemsCollectionId, resolveNativeItemsLayout, } from "../firebase/catalog-data-paths.js";
|
|
9
8
|
import { collectionRefFromSlashPath } from "../firebase/firestore-collection-path.js";
|
|
10
|
-
const MONGO_BACKUP_DB = "catalox-backups";
|
|
11
9
|
export const BACKUP_METADATA_SOURCE_NAMES = [
|
|
12
10
|
"apps",
|
|
13
11
|
"catalogs",
|
|
@@ -39,24 +37,7 @@ export function firebaseLatestName(logical) {
|
|
|
39
37
|
export function firebaseVersionedName(ts, logical) {
|
|
40
38
|
return `${ts}__backup-${logical}`;
|
|
41
39
|
}
|
|
42
|
-
|
|
43
|
-
return `__tmp_backup_${logical}`;
|
|
44
|
-
}
|
|
45
|
-
function mongoVersionedName(ts, logical) {
|
|
46
|
-
return `${ts}__${logical}`;
|
|
47
|
-
}
|
|
48
|
-
function mongoNativeItemsCollectionName(catalogId) {
|
|
49
|
-
return `catalogData__${sanitizeMongoCollectionPart(catalogId)}`;
|
|
50
|
-
}
|
|
51
|
-
function mongoSnapshotCollectionName(catalogId) {
|
|
52
|
-
return `catalogSnapshots__${sanitizeMongoCollectionPart(catalogId)}`;
|
|
53
|
-
}
|
|
54
|
-
function mongoSnapshotRunCollectionName(catalogId, runId) {
|
|
55
|
-
return `catalogSnapshotsRun__${sanitizeMongoCollectionPart(catalogId)}__${sanitizeMongoCollectionPart(runId)}`;
|
|
56
|
-
}
|
|
57
|
-
function sanitizeMongoCollectionPart(s) {
|
|
58
|
-
return String(s).replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
59
|
-
}
|
|
40
|
+
// Mongo backup mode has been removed; use explicit Mongo export/import instead.
|
|
60
41
|
export function firebaseNativeLogical(catalogId) {
|
|
61
42
|
return `catalogData--${catalogId}`;
|
|
62
43
|
}
|
|
@@ -132,203 +113,6 @@ async function copyNativeCatalogToFirestoreBackup(fs, catalogId, destCollectionP
|
|
|
132
113
|
: flatNativeItemsCollectionRef(fs, catalogId);
|
|
133
114
|
await paginatedCopyFirestoreToFirestore(fs, src.path, destCollectionPath, counts, destCollectionPath);
|
|
134
115
|
}
|
|
135
|
-
async function mongoReplacePage(col, docs) {
|
|
136
|
-
const bulk = docs.map((d) => {
|
|
137
|
-
const payload = { ...d.data() };
|
|
138
|
-
delete payload._id;
|
|
139
|
-
return {
|
|
140
|
-
replaceOne: {
|
|
141
|
-
filter: { _id: d.id },
|
|
142
|
-
replacement: { _id: d.id, ...payload },
|
|
143
|
-
upsert: true,
|
|
144
|
-
},
|
|
145
|
-
};
|
|
146
|
-
});
|
|
147
|
-
if (bulk.length)
|
|
148
|
-
await col.bulkWrite(bulk);
|
|
149
|
-
}
|
|
150
|
-
async function paginatedCopyFirestoreToMongo(fs, sourceCollectionPath, db, destCollectionName, counts, countKey) {
|
|
151
|
-
const src = fs.collection(sourceCollectionPath);
|
|
152
|
-
const col = db.collection(destCollectionName);
|
|
153
|
-
let last;
|
|
154
|
-
let total = 0;
|
|
155
|
-
while (true) {
|
|
156
|
-
let q = src.orderBy(FieldPath.documentId()).limit(200);
|
|
157
|
-
if (last)
|
|
158
|
-
q = q.startAfter(last);
|
|
159
|
-
const snap = await q.get();
|
|
160
|
-
if (snap.empty)
|
|
161
|
-
break;
|
|
162
|
-
await mongoReplacePage(col, snap.docs);
|
|
163
|
-
total += snap.docs.length;
|
|
164
|
-
last = snap.docs[snap.docs.length - 1];
|
|
165
|
-
}
|
|
166
|
-
counts[countKey] = (counts[countKey] ?? 0) + total;
|
|
167
|
-
}
|
|
168
|
-
async function promoteMongoLatest(db, tmpName, latestName) {
|
|
169
|
-
const n = await db.collection(tmpName).countDocuments();
|
|
170
|
-
if (n === 0) {
|
|
171
|
-
await db.collection(latestName).drop().catch(() => undefined);
|
|
172
|
-
return;
|
|
173
|
-
}
|
|
174
|
-
await db.collection(latestName).drop().catch(() => undefined);
|
|
175
|
-
await db.collection(tmpName).rename(latestName, { dropTarget: true });
|
|
176
|
-
}
|
|
177
|
-
async function backupMetadataMongo(fs, db, ts, counts, collectionsWritten, versionedWritten, issues) {
|
|
178
|
-
for (const name of BACKUP_METADATA_SOURCE_NAMES) {
|
|
179
|
-
const tmp = mongoTmpName(name);
|
|
180
|
-
const versioned = mongoVersionedName(ts, name);
|
|
181
|
-
try {
|
|
182
|
-
await db.collection(tmp).drop().catch(() => undefined);
|
|
183
|
-
await paginatedCopyFirestoreToMongo(fs, name, db, tmp, counts, tmp);
|
|
184
|
-
await db.collection(versioned).drop().catch(() => undefined);
|
|
185
|
-
await paginatedCopyFirestoreToMongo(fs, name, db, versioned, counts, versioned);
|
|
186
|
-
await promoteMongoLatest(db, tmp, name);
|
|
187
|
-
collectionsWritten.push(name);
|
|
188
|
-
versionedWritten.push(versioned);
|
|
189
|
-
}
|
|
190
|
-
catch (e) {
|
|
191
|
-
issues.push({
|
|
192
|
-
code: "mongo_metadata_copy_failed",
|
|
193
|
-
message: e instanceof Error ? e.message : String(e),
|
|
194
|
-
detail: { name },
|
|
195
|
-
});
|
|
196
|
-
throw e;
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
async function backupMetadataFirebase(fs, ts, counts, collectionsWritten, versionedWritten, issues) {
|
|
201
|
-
for (const name of BACKUP_METADATA_SOURCE_NAMES) {
|
|
202
|
-
const tmp = firebaseTmpName(name);
|
|
203
|
-
const latest = firebaseLatestName(name);
|
|
204
|
-
const ver = firebaseVersionedName(ts, name);
|
|
205
|
-
try {
|
|
206
|
-
await deleteAllDocsInFirestoreCollection(fs, tmp);
|
|
207
|
-
await paginatedCopyFirestoreToFirestore(fs, name, tmp, counts, tmp);
|
|
208
|
-
await paginatedCopyFirestoreToFirestore(fs, name, ver, counts, ver);
|
|
209
|
-
await deleteAllDocsInFirestoreCollection(fs, latest);
|
|
210
|
-
await paginatedCopyFirestoreToFirestore(fs, tmp, latest, counts, latest);
|
|
211
|
-
await deleteAllDocsInFirestoreCollection(fs, tmp);
|
|
212
|
-
collectionsWritten.push(latest);
|
|
213
|
-
versionedWritten.push(ver);
|
|
214
|
-
}
|
|
215
|
-
catch (e) {
|
|
216
|
-
issues.push({
|
|
217
|
-
code: "firebase_metadata_copy_failed",
|
|
218
|
-
message: e instanceof Error ? e.message : String(e),
|
|
219
|
-
detail: { name },
|
|
220
|
-
});
|
|
221
|
-
throw e;
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
async function backupSnapshotsFirebase(fs, ts, counts, collectionsWritten, versionedWritten, issues) {
|
|
226
|
-
const roots = await fs.collection("catalogSnapshots").listDocuments();
|
|
227
|
-
for (const pref of roots) {
|
|
228
|
-
const catalogId = pref.id;
|
|
229
|
-
const logical = firebaseSnapshotLogical(catalogId);
|
|
230
|
-
const itemsPath = pref.collection("items").path;
|
|
231
|
-
const tmp = firebaseTmpName(logical);
|
|
232
|
-
const latest = firebaseLatestName(logical);
|
|
233
|
-
const ver = firebaseVersionedName(ts, logical);
|
|
234
|
-
try {
|
|
235
|
-
await deleteAllDocsInFirestoreCollection(fs, tmp);
|
|
236
|
-
await paginatedCopyFirestoreToFirestore(fs, itemsPath, tmp, counts, tmp);
|
|
237
|
-
await paginatedCopyFirestoreToFirestore(fs, itemsPath, ver, counts, ver);
|
|
238
|
-
await deleteAllDocsInFirestoreCollection(fs, latest);
|
|
239
|
-
await paginatedCopyFirestoreToFirestore(fs, tmp, latest, counts, latest);
|
|
240
|
-
await deleteAllDocsInFirestoreCollection(fs, tmp);
|
|
241
|
-
collectionsWritten.push(latest);
|
|
242
|
-
versionedWritten.push(ver);
|
|
243
|
-
const runs = await pref.collection("runs").listDocuments();
|
|
244
|
-
for (const runRef of runs) {
|
|
245
|
-
const runId = runRef.id;
|
|
246
|
-
const runLogical = firebaseSnapshotRunLogical(catalogId, runId);
|
|
247
|
-
const runItemsPath = runRef.collection("items").path;
|
|
248
|
-
const rtmp = firebaseTmpName(runLogical);
|
|
249
|
-
const rlatest = firebaseLatestName(runLogical);
|
|
250
|
-
const rver = firebaseVersionedName(ts, runLogical);
|
|
251
|
-
await deleteAllDocsInFirestoreCollection(fs, rtmp);
|
|
252
|
-
await paginatedCopyFirestoreToFirestore(fs, runItemsPath, rtmp, counts, rtmp);
|
|
253
|
-
await paginatedCopyFirestoreToFirestore(fs, runItemsPath, rver, counts, rver);
|
|
254
|
-
await deleteAllDocsInFirestoreCollection(fs, rlatest);
|
|
255
|
-
await paginatedCopyFirestoreToFirestore(fs, rtmp, rlatest, counts, rlatest);
|
|
256
|
-
await deleteAllDocsInFirestoreCollection(fs, rtmp);
|
|
257
|
-
collectionsWritten.push(rlatest);
|
|
258
|
-
versionedWritten.push(rver);
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
catch (e) {
|
|
262
|
-
issues.push({
|
|
263
|
-
code: "firebase_snapshot_copy_failed",
|
|
264
|
-
message: e instanceof Error ? e.message : String(e),
|
|
265
|
-
detail: { catalogId },
|
|
266
|
-
});
|
|
267
|
-
throw e;
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
async function backupSnapshotsMongo(fs, db, ts, counts, collectionsWritten, versionedWritten, issues) {
|
|
272
|
-
const roots = await fs.collection("catalogSnapshots").listDocuments();
|
|
273
|
-
for (const pref of roots) {
|
|
274
|
-
const catalogId = pref.id;
|
|
275
|
-
const itemsPath = pref.collection("items").path;
|
|
276
|
-
const latestName = mongoSnapshotCollectionName(catalogId);
|
|
277
|
-
const versionedName = mongoVersionedName(ts, latestName);
|
|
278
|
-
const tmpName = mongoTmpName(latestName);
|
|
279
|
-
try {
|
|
280
|
-
await db.collection(tmpName).drop().catch(() => undefined);
|
|
281
|
-
await paginatedCopyFirestoreToMongo(fs, itemsPath, db, tmpName, counts, tmpName);
|
|
282
|
-
await db.collection(versionedName).drop().catch(() => undefined);
|
|
283
|
-
await paginatedCopyFirestoreToMongo(fs, itemsPath, db, versionedName, counts, versionedName);
|
|
284
|
-
await promoteMongoLatest(db, tmpName, latestName);
|
|
285
|
-
collectionsWritten.push(latestName);
|
|
286
|
-
versionedWritten.push(versionedName);
|
|
287
|
-
const runs = await pref.collection("runs").listDocuments();
|
|
288
|
-
for (const runRef of runs) {
|
|
289
|
-
const runId = runRef.id;
|
|
290
|
-
const runItemsPath = runRef.collection("items").path;
|
|
291
|
-
const runLatest = mongoSnapshotRunCollectionName(catalogId, runId);
|
|
292
|
-
const runVer = mongoVersionedName(ts, runLatest);
|
|
293
|
-
const runTmp = mongoTmpName(runLatest);
|
|
294
|
-
await db.collection(runTmp).drop().catch(() => undefined);
|
|
295
|
-
await paginatedCopyFirestoreToMongo(fs, runItemsPath, db, runTmp, counts, runTmp);
|
|
296
|
-
await db.collection(runVer).drop().catch(() => undefined);
|
|
297
|
-
await paginatedCopyFirestoreToMongo(fs, runItemsPath, db, runVer, counts, runVer);
|
|
298
|
-
await promoteMongoLatest(db, runTmp, runLatest);
|
|
299
|
-
collectionsWritten.push(runLatest);
|
|
300
|
-
versionedWritten.push(runVer);
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
catch (e) {
|
|
304
|
-
issues.push({
|
|
305
|
-
code: "mongo_snapshot_copy_failed",
|
|
306
|
-
message: e instanceof Error ? e.message : String(e),
|
|
307
|
-
detail: { catalogId },
|
|
308
|
-
});
|
|
309
|
-
throw e;
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
async function writeBackupManifestFirebase(fs, payload) {
|
|
314
|
-
await fs.collection("backup-backupRuns").add({
|
|
315
|
-
...payload,
|
|
316
|
-
createdAt: new Date().toISOString(),
|
|
317
|
-
});
|
|
318
|
-
}
|
|
319
|
-
async function writeBackupManifestMongo(db, payload) {
|
|
320
|
-
await db.collection("backupRuns").insertOne({
|
|
321
|
-
...payload,
|
|
322
|
-
createdAt: new Date().toISOString(),
|
|
323
|
-
});
|
|
324
|
-
}
|
|
325
|
-
/** Delete Mongo `backupRuns` documents with createdAt older than cutoff ISO string. */
|
|
326
|
-
export async function pruneMongoBackupRuns(db, createdBeforeIso) {
|
|
327
|
-
const res = await db.collection("backupRuns").deleteMany({
|
|
328
|
-
createdAt: { $lt: createdBeforeIso },
|
|
329
|
-
});
|
|
330
|
-
return res.deletedCount ?? 0;
|
|
331
|
-
}
|
|
332
116
|
/**
|
|
333
117
|
* Delete documents in oldest Firestore versioned backup collections matching `{ts}__backup-*`
|
|
334
118
|
* until at most `keepLast` such collections remain.
|
|
@@ -605,22 +389,6 @@ async function runBackupDataGcs(deps, input, nativeCatalogIds) {
|
|
|
605
389
|
};
|
|
606
390
|
}
|
|
607
391
|
}
|
|
608
|
-
export async function pruneFirebaseVersionedBackups(firestore, keepLast) {
|
|
609
|
-
const cols = await firestore.listCollections();
|
|
610
|
-
const versioned = cols
|
|
611
|
-
.map((c) => c.id)
|
|
612
|
-
.filter((id) => /^\d{4}_\d{2}_\d{2}T.*__backup-/.test(id))
|
|
613
|
-
.sort();
|
|
614
|
-
if (versioned.length <= keepLast)
|
|
615
|
-
return { deleted: [] };
|
|
616
|
-
const toRemove = versioned.slice(0, versioned.length - keepLast);
|
|
617
|
-
const deleted = [];
|
|
618
|
-
for (const id of toRemove) {
|
|
619
|
-
await deleteAllDocsInFirestoreCollection(firestore, id);
|
|
620
|
-
deleted.push(id);
|
|
621
|
-
}
|
|
622
|
-
return { deleted };
|
|
623
|
-
}
|
|
624
392
|
export async function runBackupData(deps, input) {
|
|
625
393
|
const issues = [];
|
|
626
394
|
const collectionsWritten = [];
|
|
@@ -636,185 +404,6 @@ export async function runBackupData(deps, input) {
|
|
|
636
404
|
...(input.catalogIds != null && input.catalogIds.length ? { catalogIds: input.catalogIds } : {}),
|
|
637
405
|
...(input.appId != null && input.appId !== "" ? { appId: input.appId } : {}),
|
|
638
406
|
});
|
|
639
|
-
|
|
640
|
-
if (!input.mongoBackupUri?.trim()) {
|
|
641
|
-
return {
|
|
642
|
-
ok: false,
|
|
643
|
-
mode: "mongo",
|
|
644
|
-
databaseName: MONGO_BACKUP_DB,
|
|
645
|
-
backupTimestamp: new Date().toISOString(),
|
|
646
|
-
...(input.backupLabel != null ? { backupLabel: input.backupLabel } : {}),
|
|
647
|
-
includeSnapshots,
|
|
648
|
-
collectionsWritten: [],
|
|
649
|
-
versionedCollectionsWritten: [],
|
|
650
|
-
counts: {},
|
|
651
|
-
issues: [{ code: "missing_mongo_uri", message: "mongoBackupUri is required for mongo mode" }],
|
|
652
|
-
nativeItemSourceLayoutByCatalogId: {},
|
|
653
|
-
};
|
|
654
|
-
}
|
|
655
|
-
const client = new MongoClient(input.mongoBackupUri);
|
|
656
|
-
try {
|
|
657
|
-
await client.connect();
|
|
658
|
-
const db = client.db(MONGO_BACKUP_DB);
|
|
659
|
-
await backupMetadataMongo(fs, db, ts, counts, collectionsWritten, versionedCollectionsWritten, issues);
|
|
660
|
-
for (const catalogId of nativeCatalogIds) {
|
|
661
|
-
const layout = await resolveNativeItemsLayout(fs, catalogId);
|
|
662
|
-
nativeItemSourceLayoutByCatalogId[String(catalogId)] = layout;
|
|
663
|
-
const logical = mongoNativeItemsCollectionName(String(catalogId));
|
|
664
|
-
const tmp = mongoTmpName(logical);
|
|
665
|
-
const versioned = mongoVersionedName(ts, logical);
|
|
666
|
-
await db.collection(tmp).drop().catch(() => undefined);
|
|
667
|
-
await copyNativeCatalogToMongo(fs, db, catalogId, tmp, counts, layout);
|
|
668
|
-
await db.collection(versioned).drop().catch(() => undefined);
|
|
669
|
-
await copyNativeCatalogToMongo(fs, db, catalogId, versioned, counts, layout);
|
|
670
|
-
await promoteMongoLatest(db, tmp, logical);
|
|
671
|
-
collectionsWritten.push(logical);
|
|
672
|
-
versionedCollectionsWritten.push(versioned);
|
|
673
|
-
}
|
|
674
|
-
if (includeSnapshots) {
|
|
675
|
-
await backupSnapshotsMongo(fs, db, ts, counts, collectionsWritten, versionedCollectionsWritten, issues);
|
|
676
|
-
}
|
|
677
|
-
await writeBackupManifestMongo(db, {
|
|
678
|
-
backupTimestamp: new Date().toISOString(),
|
|
679
|
-
mode: "mongo",
|
|
680
|
-
backupLabel: input.backupLabel,
|
|
681
|
-
includeSnapshots,
|
|
682
|
-
collectionsWritten,
|
|
683
|
-
versionedCollectionsWritten,
|
|
684
|
-
counts,
|
|
685
|
-
issues,
|
|
686
|
-
nativeItemSourceLayoutByCatalogId,
|
|
687
|
-
});
|
|
688
|
-
return {
|
|
689
|
-
ok: true,
|
|
690
|
-
mode: "mongo",
|
|
691
|
-
databaseName: MONGO_BACKUP_DB,
|
|
692
|
-
backupTimestamp: new Date().toISOString(),
|
|
693
|
-
...(input.backupLabel != null ? { backupLabel: input.backupLabel } : {}),
|
|
694
|
-
includeSnapshots,
|
|
695
|
-
collectionsWritten,
|
|
696
|
-
versionedCollectionsWritten,
|
|
697
|
-
counts,
|
|
698
|
-
issues,
|
|
699
|
-
nativeItemSourceLayoutByCatalogId,
|
|
700
|
-
};
|
|
701
|
-
}
|
|
702
|
-
catch (e) {
|
|
703
|
-
issues.push({
|
|
704
|
-
code: "mongo_backup_failed",
|
|
705
|
-
message: e instanceof Error ? e.message : String(e),
|
|
706
|
-
});
|
|
707
|
-
return {
|
|
708
|
-
ok: false,
|
|
709
|
-
mode: "mongo",
|
|
710
|
-
databaseName: MONGO_BACKUP_DB,
|
|
711
|
-
backupTimestamp: new Date().toISOString(),
|
|
712
|
-
...(input.backupLabel != null ? { backupLabel: input.backupLabel } : {}),
|
|
713
|
-
includeSnapshots,
|
|
714
|
-
collectionsWritten,
|
|
715
|
-
versionedCollectionsWritten,
|
|
716
|
-
counts,
|
|
717
|
-
issues,
|
|
718
|
-
nativeItemSourceLayoutByCatalogId,
|
|
719
|
-
};
|
|
720
|
-
}
|
|
721
|
-
finally {
|
|
722
|
-
await client.close().catch(() => undefined);
|
|
723
|
-
}
|
|
724
|
-
}
|
|
725
|
-
if (input.mode === "gcs") {
|
|
726
|
-
return await runBackupDataGcs(deps, input, nativeCatalogIds);
|
|
727
|
-
}
|
|
728
|
-
if (input.mode !== "firebase") {
|
|
729
|
-
return {
|
|
730
|
-
ok: false,
|
|
731
|
-
mode: input.mode,
|
|
732
|
-
databaseName: "",
|
|
733
|
-
backupTimestamp: new Date().toISOString(),
|
|
734
|
-
...(input.backupLabel != null ? { backupLabel: input.backupLabel } : {}),
|
|
735
|
-
includeSnapshots,
|
|
736
|
-
collectionsWritten: [],
|
|
737
|
-
versionedCollectionsWritten: [],
|
|
738
|
-
counts: {},
|
|
739
|
-
issues: [
|
|
740
|
-
{
|
|
741
|
-
code: "unknown_backup_mode",
|
|
742
|
-
message: `Unsupported backup mode: ${String(input.mode)}`,
|
|
743
|
-
},
|
|
744
|
-
],
|
|
745
|
-
nativeItemSourceLayoutByCatalogId: {},
|
|
746
|
-
};
|
|
747
|
-
}
|
|
748
|
-
try {
|
|
749
|
-
await backupMetadataFirebase(fs, ts, counts, collectionsWritten, versionedCollectionsWritten, issues);
|
|
750
|
-
for (const catalogId of nativeCatalogIds) {
|
|
751
|
-
const layout = await resolveNativeItemsLayout(fs, catalogId);
|
|
752
|
-
nativeItemSourceLayoutByCatalogId[String(catalogId)] = layout;
|
|
753
|
-
const logical = firebaseNativeLogical(String(catalogId));
|
|
754
|
-
const tmp = firebaseTmpName(logical);
|
|
755
|
-
const latest = firebaseLatestName(logical);
|
|
756
|
-
const ver = firebaseVersionedName(ts, logical);
|
|
757
|
-
await deleteAllDocsInFirestoreCollection(fs, tmp);
|
|
758
|
-
await copyNativeCatalogToFirestoreBackup(fs, catalogId, tmp, counts, layout);
|
|
759
|
-
await copyNativeCatalogToFirestoreBackup(fs, catalogId, ver, counts, layout);
|
|
760
|
-
await deleteAllDocsInFirestoreCollection(fs, latest);
|
|
761
|
-
await paginatedCopyFirestoreToFirestore(fs, tmp, latest, counts, latest);
|
|
762
|
-
await deleteAllDocsInFirestoreCollection(fs, tmp);
|
|
763
|
-
collectionsWritten.push(latest);
|
|
764
|
-
versionedCollectionsWritten.push(ver);
|
|
765
|
-
}
|
|
766
|
-
if (includeSnapshots) {
|
|
767
|
-
await backupSnapshotsFirebase(fs, ts, counts, collectionsWritten, versionedCollectionsWritten, issues);
|
|
768
|
-
}
|
|
769
|
-
await writeBackupManifestFirebase(fs, {
|
|
770
|
-
backupTimestamp: new Date().toISOString(),
|
|
771
|
-
mode: "firebase",
|
|
772
|
-
backupLabel: input.backupLabel,
|
|
773
|
-
includeSnapshots,
|
|
774
|
-
collectionsWritten,
|
|
775
|
-
versionedCollectionsWritten,
|
|
776
|
-
counts,
|
|
777
|
-
issues,
|
|
778
|
-
nativeItemSourceLayoutByCatalogId,
|
|
779
|
-
});
|
|
780
|
-
return {
|
|
781
|
-
ok: true,
|
|
782
|
-
mode: "firebase",
|
|
783
|
-
databaseName: "(same-firestore-target)",
|
|
784
|
-
backupTimestamp: new Date().toISOString(),
|
|
785
|
-
...(input.backupLabel != null ? { backupLabel: input.backupLabel } : {}),
|
|
786
|
-
includeSnapshots,
|
|
787
|
-
collectionsWritten,
|
|
788
|
-
versionedCollectionsWritten,
|
|
789
|
-
counts,
|
|
790
|
-
issues,
|
|
791
|
-
nativeItemSourceLayoutByCatalogId,
|
|
792
|
-
};
|
|
793
|
-
}
|
|
794
|
-
catch (e) {
|
|
795
|
-
issues.push({
|
|
796
|
-
code: "firebase_backup_failed",
|
|
797
|
-
message: e instanceof Error ? e.message : String(e),
|
|
798
|
-
});
|
|
799
|
-
return {
|
|
800
|
-
ok: false,
|
|
801
|
-
mode: "firebase",
|
|
802
|
-
databaseName: "(same-firestore-target)",
|
|
803
|
-
backupTimestamp: new Date().toISOString(),
|
|
804
|
-
...(input.backupLabel != null ? { backupLabel: input.backupLabel } : {}),
|
|
805
|
-
includeSnapshots,
|
|
806
|
-
collectionsWritten,
|
|
807
|
-
versionedCollectionsWritten,
|
|
808
|
-
counts,
|
|
809
|
-
issues,
|
|
810
|
-
nativeItemSourceLayoutByCatalogId,
|
|
811
|
-
};
|
|
812
|
-
}
|
|
813
|
-
}
|
|
814
|
-
async function copyNativeCatalogToMongo(fs, db, catalogId, destCollection, counts, layout) {
|
|
815
|
-
const src = layout === "legacy"
|
|
816
|
-
? legacyNativeItemsCollectionRef(fs, catalogId)
|
|
817
|
-
: flatNativeItemsCollectionRef(fs, catalogId);
|
|
818
|
-
await paginatedCopyFirestoreToMongo(fs, src.path, db, destCollection, counts, destCollection);
|
|
407
|
+
return await runBackupDataGcs(deps, input, nativeCatalogIds);
|
|
819
408
|
}
|
|
820
409
|
//# sourceMappingURL=backup-data.js.map
|