@x12i/catalox 2.2.2 → 2.4.0

package/README.md

@@ -22,7 +22,7 @@ This repo is currently set up as a workspace package.
 
 ## Configuration (real connections)
 
-Catalox is a library: you provide initialized clients + runtime env.
+Catalox is a library: you provide initialized clients + runtime env. For a **full list of environment variables**, CLI vs library behavior, and integration-test requirements, see [`docs/environment.md`](docs/environment.md).
 
 ### Firestore (Firebase Admin SDK)
 
@@ -66,15 +66,39 @@ GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
 FIREBASE_SERVICE_ACCOUNT_PATH=/path/to/service-account.json
 FIREBASE_PROJECT_ID=your-project-id
 FIRESTORE_LIVE_TESTS=1
+# Optional defaults for the packaged `catalox` CLI (Catalox app/store context, not GCP):
+# CATALOX_APP_ID=myAppId
+# CATALOX_STORE_ID=myStoreId
+# CATALOX_USER_ID=user-123
+# CATALOX_MONGO_URI=mongodb://127.0.0.1:27017
+```
+
+### `catalox` CLI environment
+
+The `catalox` binary loads `.env` via `dotenv`. Use **`CATALOX_*`** for **Catalox runtime context** (catalog `appId`, optional `storeId`, actor, Mongo URI for the bundled adapter). Use **`FIREBASE_*` / `FIRESTORE_*`** for **Firebase Admin / Firestore connection** (service account path, GCP `projectId`, database id, live-test flags).
+
+- **`CATALOX_APP_ID`** — Default `appId` in CLI context when a command does not pass `--app` (when present, `--app` overrides).
+- **`CATALOX_STORE_ID`** — Default `storeId` for **`report`** and **`export`** when `--store` is omitted (`--store` overrides).
+- **`CATALOX_USER_ID`** — Optional user id / actor for authz-sensitive CLI paths.
+- **`CATALOX_MONGO_URI`** — If set, enables the Mongo catalog adapter in the CLI (see `createCatalox()` in `src/cli/index.ts`).
+
+**Running commands from this repository:** the shell command `catalox` is only on your `PATH` if the package is installed globally (`npm i -g @x12i/catalox`) or linked (`npm link` from this repo). Otherwise, after `npm run build`, use:
+
+```bash
+npm run cli -- firestore report-native-layout --app "narrix"
+# same as: node dist/src/cli/index.js firestore report-native-layout --app "narrix"
 ```
 
 ## Firestore data model (logical collections)
 
+For a **full layout** (subcollections, document id conventions, snapshot runs, and query notes), see [`docs/firestore-data-model.md`](docs/firestore-data-model.md).
+
 Metadata:
 
 - `apps/{appId}`
 - `catalogs/{catalogId}`
 - `catalogBindings/{bindingId}` (many-to-many app↔catalog)
+- `storeAppBindings/{bindingId}` (store↔app, multi-app export/report)
 - `catalogDefinitions/{catalogId}` (native vs mapped specifics)
 - `catalogAdapters/{adapterId}` (mongo/api adapter definitions)
 - `catalogMappings/{mappingId}` (field mapping specs)
@@ -84,16 +108,24 @@ Metadata:
 
 Data:
 
-- `catalogData/{catalogId}/items/{itemId}` (native items)
+- `catalogData-{catalogId}-items/{itemId}` (native item rows; top-level collection per catalog)
+- `catalogData/{catalogId}` (index and metadata for that catalog’s native storage; not item payloads)
 - `catalogSnapshots/{catalogId}/items/{itemId}` (mapped snapshot mode)
 
+Backups (optional operator feature):
+
+- `backup-*` and `{timestamp}__backup-*` (Firebase mode, same database)
+- Mongo database `catalox-backups` (Mongo mode)
+
+See [`docs/backup.md`](docs/backup.md) for `backupData` / CLI backup, [`docs/restore-firestore-backup.md`](docs/restore-firestore-backup.md) for restoring Firebase mirrors to live data with undo, [`docs/migration-native-catalog-data.md`](docs/migration-native-catalog-data.md) for moving legacy native rows into `catalogData-{catalogId}-items`, and [`docs/firestore-native-layout-vs-backup.md`](docs/firestore-native-layout-vs-backup.md) for how live paths differ from `backup-*` mirrors and what `backupData` reports in `nativeItemSourceLayoutByCatalogId`. If the console still shows **`catalogData/{catalogId}/items`**, run **`catalox firestore report-native-layout`** then **`catalox firestore migrate-native-catalog-data`** (backup + copy to flat; optional `--delete-legacy` after you trust backups).
+
 ## Core usage (generic from `appId`)
 
 ### Create stores and `Catalox`
 
 ```ts
 import { FirestoreStore } from "@x12i/catalox";
-import { AppStore, CatalogStore, BindingStore, DefinitionStore, MappingStore, DescriptorStore, ReferenceStore, NativeItemStore, SnapshotStore, AdapterStore } from "@x12i/catalox";
+import { AppStore, CatalogStore, BindingStore, DefinitionStore, MappingStore, DescriptorStore, ReferenceStore, NativeItemStore, CatalogDataIndexStore, SnapshotStore, AdapterStore } from "@x12i/catalox";
 import { AuthorizationService, Catalox } from "@x12i/catalox";
 
 // firebase-admin initialization is up to the consumer
@@ -102,18 +134,21 @@ import { getFirestore } from "firebase-admin/firestore";
 const firestore = getFirestore();
 const store = new FirestoreStore({ firestore });
 
+const bindings = new BindingStore(store);
 const deps = {
   apps: new AppStore(store),
   catalogs: new CatalogStore(store),
-  bindings: new BindingStore(store),
+  bindings,
   definitions: new DefinitionStore(store),
   mappings: new MappingStore(store),
   descriptors: new DescriptorStore(store),
   references: new ReferenceStore(store),
   nativeItems: new NativeItemStore(store),
+  catalogDataIndex: new CatalogDataIndexStore(store),
   snapshots: new SnapshotStore(store),
   adapters: new AdapterStore(store),
-  authz: new AuthorizationService(new BindingStore(store)),
+  firestoreStore: store,
+  authz: new AuthorizationService(bindings),
 };
 
 const catalox = new Catalox(deps);
@@ -306,7 +341,7 @@ await catalox.createCatalog(ctx, {
   catalogId: "signals",
   name: "Signals",
   sourceMode: "native",
-  native: { type: "native", firestoreCollectionPath: "catalogData/signals/items" },
+  native: { type: "native" },
 });
 ```
 
@@ -450,7 +485,7 @@ Integration tests are **live-only** (no mocks, no emulators). They require:
 ### Live test safety (read before running)
 
 - **Never run against production credentials/projects.** Use a dedicated Firebase project for tests.
-- **Touched collections**: `apps`, `catalogs`, `catalogBindings`, `catalogDefinitions`, `catalogDescriptors`, and `catalogData/{catalogId}/items/...`.
+- **Touched collections**: `apps`, `catalogs`, `catalogBindings`, `catalogDefinitions`, `catalogDescriptors`, `catalogData/{catalogId}`, and `catalogData-{catalogId}-items/...`. The **`catalox firestore restore-backup`** / **`undo-restore-backup`** commands additionally write `backup-restoreSessions` and `{restoreSessionId}__preRestore-*` sidecar collections (see [`docs/restore-firestore-backup.md`](docs/restore-firestore-backup.md)).
 - **Cleanup**: tests do best-effort deletes of the docs they created, but do not guarantee full teardown.
 
 ## Boundaries (important)

package/dist/src/catalox/backup-data.d.ts

@@ -0,0 +1,40 @@
+import { type Firestore } from "firebase-admin/firestore";
+import { type Db } from "mongodb";
+import type { CatalogId } from "../contracts/ids.js";
+import type { BackupDataInput, BackupDataResult } from "../contracts/backup.js";
+import type { CatalogRecord } from "../contracts/catalogs.js";
+import { CatalogStore } from "../firebase/catalog-store.js";
+import { BindingStore } from "../firebase/binding-store.js";
+import { FirestoreStore } from "../firebase/firestore-store.js";
+export declare const BACKUP_METADATA_SOURCE_NAMES: readonly ["apps", "catalogs", "catalogBindings", "storeAppBindings", "catalogDefinitions", "catalogDescriptors", "catalogMappings", "catalogAdapters", "catalogReferences", "catalogData"];
+export type BackupDataDeps = {
+    firestoreStore: FirestoreStore;
+    catalogs: CatalogStore;
+    bindings: BindingStore;
+};
+export declare function formatBackupTimestamp(versionOverride?: string, d?: Date): string;
+/** Latest Firebase backup mirror collection id for a logical source (e.g. `apps`, `catalogData--{id}`). */
+export declare function firebaseLatestName(logical: string): string;
+/** Versioned Firebase backup collection id. */
+export declare function firebaseVersionedName(ts: string, logical: string): string;
+export declare function firebaseNativeLogical(catalogId: string): string;
+export declare function firebaseSnapshotLogical(catalogId: string): string;
+export declare function firebaseSnapshotRunLogical(catalogId: string, runId: string): string;
+export declare function paginatedCopyFirestoreToFirestore(fs: Firestore, sourceCollectionPath: string, destCollectionPath: string, counts: Record<string, number>, countKey: string): Promise<void>;
+export declare function deleteAllDocsInFirestoreCollection(fs: Firestore, collectionPath: string): Promise<void>;
+export declare function discoverNativeCatalogIds(fs: Firestore, catalogs: CatalogRecord[], opts: {
+    bindings: BindingStore;
+    catalogIds?: CatalogId[];
+    appId?: string;
+}): Promise<CatalogId[]>;
+/** Delete Mongo `backupRuns` documents with createdAt older than cutoff ISO string. */
+export declare function pruneMongoBackupRuns(db: Db, createdBeforeIso: string): Promise<number>;
+/**
+ * Delete documents in oldest Firestore versioned backup collections matching `{ts}__backup-*`
+ * until at most `keepLast` such collections remain.
+ */
+export declare function pruneFirebaseVersionedBackups(firestore: Firestore, keepLast: number): Promise<{
+    deleted: string[];
+}>;
+export declare function runBackupData(deps: BackupDataDeps, input: BackupDataInput): Promise<BackupDataResult>;
+//# sourceMappingURL=backup-data.d.ts.map

package/dist/src/catalox/backup-data.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backup-data.d.ts","sourceRoot":"","sources":["../../../src/catalox/backup-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,KAAK,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EAAe,KAAK,EAAE,EAAE,MAAM,SAAS,CAAC;AAE/C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAmB,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AACjG,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAWhE,eAAO,MAAM,4BAA4B,4LAY/B,CAAC;AAEX,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,EAAE,YAAY,CAAC;IACvB,QAAQ,EAAE,YAAY,CAAC;CACxB,CAAC;AAEF,wBAAgB,qBAAqB,CAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC,OAAa,GAAG,MAAM,CAMtF;AAMD,2GAA2G;AAC3G,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,+CAA+C;AAC/C,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAEzE;AA0BD,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAE/D;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjE;AAED,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnF;AAED,wBAAsB,iCAAiC,CACrD,EAAE,EAAE,SAAS,EACb,oBAAoB,EAAE,MAAM,EAC5B,kBAAkB,EAAE,MAAM,EAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC9B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED,wBAAsB,kCAAkC,CAAC,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAS7G;AAED,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,aAAa,EAAE,EACzB,IAAI,EAAE;IAAE,QAAQ,EAAE,YAAY,CAAC;IAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACzE,OAAO,CAAC,SAAS,EAAE,CAAC,CAwBtB;AA0PD,uFAAuF;AACvF,wBAAsB,oBAAoB,CAAC,EAAE,EAAE,EAAE,EAAE,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAK5F;AAED;;;GAGG;AACH,wBAAsB,6BAA6B,CACjD,SAAS,EAAE,SAAS,EACpB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAchC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,cAAc,EACpB,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,gBAAgB,CAAC,CA+K3B"}