@lpdjs/firestore-repo-service 2.1.2 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -13
- package/dist/index.cjs +44 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +44 -44
- package/dist/index.js.map +1 -1
- package/dist/servers/admin/index.cjs +4 -4
- package/dist/servers/admin/index.cjs.map +1 -1
- package/dist/servers/admin/index.js +4 -4
- package/dist/servers/admin/index.js.map +1 -1
- package/dist/servers/crud/index.cjs +2 -2
- package/dist/servers/crud/index.cjs.map +1 -1
- package/dist/servers/crud/index.js +2 -2
- package/dist/servers/crud/index.js.map +1 -1
- package/dist/servers/index.cjs +2 -2
- package/dist/servers/index.cjs.map +1 -1
- package/dist/servers/index.js +2 -2
- package/dist/servers/index.js.map +1 -1
- package/dist/sync/bigquery.d.cts +1 -1
- package/dist/sync/bigquery.d.ts +1 -1
- package/dist/sync/index.cjs +33 -33
- package/dist/sync/index.cjs.map +1 -1
- package/dist/sync/index.d.cts +170 -170
- package/dist/sync/index.d.ts +170 -170
- package/dist/sync/index.js +33 -33
- package/dist/sync/index.js.map +1 -1
- package/dist/{types-PzZ0APQ_.d.cts → types-BG1kGsLO.d.cts} +25 -12
- package/dist/{types-PzZ0APQ_.d.ts → types-BG1kGsLO.d.ts} +25 -12
- package/package.json +1 -1
package/dist/sync/index.d.ts
CHANGED
|
@@ -1,70 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export {
|
|
1
|
+
import { S as SqlAdapter, a as SyncEvent, b as adminsyncConfig, R as RepoSyncConfig, P as PubSubClientDep, F as FirestoreSyncConfig, c as SqlDialect, d as SqlColumn, e as SqlTableDef, G as GenerateDDLConfig, L as LogicalType, f as SyncTriggersConfig, g as SyncWorkerConfig } from '../types-BG1kGsLO.js';
|
|
2
|
+
export { h as FirestoreTriggersDep, O as OrFactory, i as PubSubHandlerDep, j as SyncDeps, k as SyncOperation, l as adminsyncBasicAuth, m as adminsyncFeaturesFlag } from '../types-BG1kGsLO.js';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
|
|
5
|
-
declare function zodTypeToLogical(schema: z.ZodType): LogicalType;
|
|
6
|
-
interface ZodSchemaToColumnsOptions {
|
|
7
|
-
primaryKey?: string;
|
|
8
|
-
exclude?: string[];
|
|
9
|
-
columnMap?: Record<string, string>;
|
|
10
|
-
}
|
|
11
|
-
/**
|
|
12
|
-
* Convert a Zod object schema into an array of {@link SqlColumn} definitions
|
|
13
|
-
* suitable for SQL table creation.
|
|
14
|
-
*
|
|
15
|
-
* Nested ZodObject fields are recursively flattened into separate columns
|
|
16
|
-
* with underscore-separated names (e.g. `address.street` → `address_street`).
|
|
17
|
-
* Arrays become JSON columns.
|
|
18
|
-
*/
|
|
19
|
-
declare function zodSchemaToColumns(schema: z.ZodObject<any>, dialect: SqlDialect, options?: ZodSchemaToColumnsOptions): SqlColumn[];
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Convert a single Firestore value into a SQL-safe primitive.
|
|
23
|
-
*
|
|
24
|
-
* Complex types (arrays, GeoPoints, binary) become JSON strings.
|
|
25
|
-
* Primitives pass through unchanged.
|
|
26
|
-
* Objects are NOT stringified here — they are flattened by serializeDocument.
|
|
27
|
-
*/
|
|
28
|
-
declare function serializeValue(value: unknown): unknown;
|
|
29
|
-
/**
|
|
30
|
-
* Serialize a full Firestore document into a flat object of SQL-safe values.
|
|
31
|
-
*
|
|
32
|
-
* Nested objects are flattened into underscore-separated column names
|
|
33
|
-
* (e.g. `address.street` → `address_street`). Arrays become JSON strings.
|
|
34
|
-
* Applies optional field exclusions and column renames from `options`.
|
|
35
|
-
*/
|
|
36
|
-
declare function serializeDocument(doc: Record<string, unknown>, options?: Pick<RepoSyncConfig, "exclude" | "columnMap">): Record<string, unknown>;
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* DDL generator — produces CREATE TABLE / ALTER TABLE statements from
|
|
40
|
-
* SqlColumn definitions and a SqlDialect.
|
|
41
|
-
*
|
|
42
|
-
* `generateDDL()` is the public entry point: it walks a repository mapping,
|
|
43
|
-
* converts each repo's Zod schema to columns, and returns the full DDL
|
|
44
|
-
* as a single string.
|
|
45
|
-
*/
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Generate a CREATE TABLE statement from a table definition.
|
|
49
|
-
* Delegates to the dialect for syntax specifics.
|
|
50
|
-
*/
|
|
51
|
-
declare function createTableDDL(dialect: SqlDialect, table: SqlTableDef): string;
|
|
52
|
-
/**
|
|
53
|
-
* Generate ALTER TABLE ADD COLUMN statements for columns missing from an
|
|
54
|
-
* existing table.
|
|
55
|
-
*/
|
|
56
|
-
declare function addColumnsDDL(dialect: SqlDialect, tableName: string, columns: SqlColumn[]): string;
|
|
57
|
-
/**
|
|
58
|
-
* Walk a full repository mapping and produce DDL for every repo that has a
|
|
59
|
-
* Zod schema attached.
|
|
60
|
-
*
|
|
61
|
-
* @param repoMapping - Object whose values expose `.schema` (ZodObject)
|
|
62
|
-
* @param dialect - Target SQL dialect
|
|
63
|
-
* @param config - Optional per-repo overrides (table name, exclusions…)
|
|
64
|
-
* @returns Complete DDL string (one CREATE TABLE per repo, separated by newlines)
|
|
65
|
-
*/
|
|
66
|
-
declare function generateDDL<M extends Record<string, any>>(repoMapping: M, dialect: SqlDialect, config?: GenerateDDLConfig<NoInfer<M>>): string;
|
|
67
|
-
|
|
68
5
|
/**
|
|
69
6
|
* Per-repo in-memory batch buffer.
|
|
70
7
|
*
|
|
@@ -115,35 +52,127 @@ declare class SyncQueue {
|
|
|
115
52
|
}
|
|
116
53
|
|
|
117
54
|
/**
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
* flushes batches to the configured {@link SqlAdapter}.
|
|
55
|
+
* Sync Admin — optional HTTP endpoint for inspecting and managing the
|
|
56
|
+
* Firestore → SQL sync pipeline.
|
|
121
57
|
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
58
|
+
* Features (gated by `featuresFlag`):
|
|
59
|
+
* - **healthCheck** — compare expected Zod-derived columns vs actual SQL columns
|
|
60
|
+
* - **manualSync** — force re-sync all documents in a Firestore collection
|
|
61
|
+
* - **viewQueue** — inspect pending items in the per-repo SyncQueue
|
|
62
|
+
*
|
|
63
|
+
* @example
|
|
64
|
+
* ```typescript
|
|
65
|
+
* const sync = createFirestoreSync(repos, {
|
|
66
|
+
* // …deps, adapter, etc.
|
|
67
|
+
* admin: {
|
|
68
|
+
* auth: { type: "basic", username: "admin", password: "secret" },
|
|
69
|
+
* featuresFlag: { healthCheck: true, manualSync: true, viewQueue: true },
|
|
70
|
+
* },
|
|
71
|
+
* });
|
|
72
|
+
*
|
|
73
|
+
* export const adminsync = onRequest(sync.adminHandler!);
|
|
74
|
+
* ```
|
|
124
75
|
*/
|
|
125
76
|
|
|
126
77
|
/**
|
|
127
|
-
* Create
|
|
128
|
-
* to a SQL database.
|
|
78
|
+
* Create the sync admin HTTP handler.
|
|
129
79
|
*
|
|
130
|
-
*
|
|
131
|
-
* -
|
|
132
|
-
* -
|
|
133
|
-
*
|
|
134
|
-
* -
|
|
80
|
+
* @param repoMapping - The configured repository mapping
|
|
81
|
+
* @param adapter - The SQL adapter (e.g. BigQueryAdapter)
|
|
82
|
+
* @param queues - Live queue map from the worker
|
|
83
|
+
* @param handleMessage - Direct SyncEvent processor from the worker
|
|
84
|
+
* @param config - Admin-specific config (auth, basePath, features)
|
|
85
|
+
* @param repoConfigs - Per-repo sync config (tableName, exclude, columnMap…)
|
|
86
|
+
* @param pubsub - PubSub client (needed for configCheck)
|
|
87
|
+
* @param topicPrefix - PubSub topic prefix (needed for configCheck)
|
|
135
88
|
*/
|
|
136
|
-
declare function
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
89
|
+
declare function createadminsyncServer(repoMapping: Record<string, any>, adapter: SqlAdapter, queues: Map<string, SyncQueue>, handleMessage: (event: SyncEvent) => Promise<void>, config: adminsyncConfig, repoConfigs: Record<string, RepoSyncConfig<string> | undefined>, pubsub?: PubSubClientDep, topicPrefix?: string): (req: any, res: any) => Promise<void>;
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Unified wrapper — combines triggers + worker into a single call.
|
|
93
|
+
*
|
|
94
|
+
* @example
|
|
95
|
+
* ```typescript
|
|
96
|
+
* import * as firestoreTriggers from "firebase-functions/v2/firestore";
|
|
97
|
+
* import * as pubsubHandler from "firebase-functions/v2/pubsub";
|
|
98
|
+
* import { PubSub } from "@google-cloud/pubsub";
|
|
99
|
+
*
|
|
100
|
+
* const sync = createFirestoreSync(repos, {
|
|
101
|
+
* deps: { firestoreTriggers, pubsubHandler, pubsub: new PubSub() },
|
|
102
|
+
* adapter,
|
|
103
|
+
* topicPrefix: "firestore-sync",
|
|
104
|
+
* autoMigrate: true,
|
|
105
|
+
* admin: {
|
|
106
|
+
* auth: { type: "basic", username: "admin", password: "secret" },
|
|
107
|
+
* featuresFlag: { healthCheck: true, manualSync: true, viewQueue: true },
|
|
108
|
+
* },
|
|
109
|
+
* repos: {
|
|
110
|
+
* users: { exclude: ["documentPath"], columnMap: { docId: "user_id" } },
|
|
111
|
+
* posts: { columnMap: { docId: "post_id" } },
|
|
112
|
+
* },
|
|
113
|
+
* });
|
|
114
|
+
*
|
|
115
|
+
* // Triggers + PubSub handlers
|
|
116
|
+
* export const { users_onCreate, users_onUpdate, users_onDelete, sync_users } = sync.functions;
|
|
117
|
+
*
|
|
118
|
+
* // Admin endpoint — wrap with onRequest yourself
|
|
119
|
+
* export const adminsync = onRequest(sync.adminHandler!);
|
|
120
|
+
*
|
|
121
|
+
* // Or pass onRequest in admin config to auto-add to sync.functions:
|
|
122
|
+
* // admin: { onRequest, ... } → export const { adminsync } = sync.functions;
|
|
123
|
+
* ```
|
|
124
|
+
*/
|
|
125
|
+
|
|
126
|
+
declare function createFirestoreSync<M extends Record<string, any>>(repoMapping: M, config: FirestoreSyncConfig<NoInfer<M>>): {
|
|
127
|
+
/** All Cloud Functions (triggers + handlers + optional admin) — spread into exports */
|
|
128
|
+
functions: Record<string, any>;
|
|
129
|
+
/**
|
|
130
|
+
* Raw admin HTTP handler — wrap with `onRequest()` yourself if you
|
|
131
|
+
* didn't pass `onRequest` in the admin config.
|
|
132
|
+
* @example
|
|
133
|
+
* ```ts
|
|
134
|
+
* export const adminsync = onRequest(sync.adminHandler!);
|
|
135
|
+
* ```
|
|
136
|
+
*/
|
|
137
|
+
adminHandler: ((req: any, res: any) => Promise<void>) | null;
|
|
138
|
+
/** Process a SyncEvent directly (for testing) */
|
|
139
|
+
handleMessage: (event: SyncEvent) => Promise<void>;
|
|
140
|
+
/** Internal queue map (for testing) */
|
|
142
141
|
queues: Map<string, SyncQueue>;
|
|
143
|
-
/** Flush all queues and stop timers
|
|
144
|
-
shutdown()
|
|
142
|
+
/** Flush all queues and stop timers */
|
|
143
|
+
shutdown: () => Promise<void>;
|
|
145
144
|
};
|
|
146
145
|
|
|
146
|
+
/**
|
|
147
|
+
* DDL generator — produces CREATE TABLE / ALTER TABLE statements from
|
|
148
|
+
* SqlColumn definitions and a SqlDialect.
|
|
149
|
+
*
|
|
150
|
+
* `generateDDL()` is the public entry point: it walks a repository mapping,
|
|
151
|
+
* converts each repo's Zod schema to columns, and returns the full DDL
|
|
152
|
+
* as a single string.
|
|
153
|
+
*/
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Generate a CREATE TABLE statement from a table definition.
|
|
157
|
+
* Delegates to the dialect for syntax specifics.
|
|
158
|
+
*/
|
|
159
|
+
declare function createTableDDL(dialect: SqlDialect, table: SqlTableDef): string;
|
|
160
|
+
/**
|
|
161
|
+
* Generate ALTER TABLE ADD COLUMN statements for columns missing from an
|
|
162
|
+
* existing table.
|
|
163
|
+
*/
|
|
164
|
+
declare function addColumnsDDL(dialect: SqlDialect, tableName: string, columns: SqlColumn[]): string;
|
|
165
|
+
/**
|
|
166
|
+
* Walk a full repository mapping and produce DDL for every repo that has a
|
|
167
|
+
* Zod schema attached.
|
|
168
|
+
*
|
|
169
|
+
* @param repoMapping - Object whose values expose `.schema` (ZodObject)
|
|
170
|
+
* @param dialect - Target SQL dialect
|
|
171
|
+
* @param config - Optional per-repo overrides (table name, exclusions…)
|
|
172
|
+
* @returns Complete DDL string (one CREATE TABLE per repo, separated by newlines)
|
|
173
|
+
*/
|
|
174
|
+
declare function generateDDL<M extends Record<string, any>>(repoMapping: M, dialect: SqlDialect, config?: GenerateDDLConfig<NoInfer<M>>): string;
|
|
175
|
+
|
|
147
176
|
/**
|
|
148
177
|
* Migration manager — generates DDL and optionally auto-migrates SQL tables
|
|
149
178
|
* to match the Zod schemas defined in a repository mapping.
|
|
@@ -169,6 +198,39 @@ interface MigrateResult {
|
|
|
169
198
|
*/
|
|
170
199
|
declare function autoMigrate<M extends Record<string, any>>(repoMapping: M, adapter: SqlAdapter, config?: GenerateDDLConfig<NoInfer<M>>): Promise<MigrateResult>;
|
|
171
200
|
|
|
201
|
+
declare function zodTypeToLogical(schema: z.ZodType): LogicalType;
|
|
202
|
+
interface ZodSchemaToColumnsOptions {
|
|
203
|
+
primaryKey?: string;
|
|
204
|
+
exclude?: string[];
|
|
205
|
+
columnMap?: Record<string, string>;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Convert a Zod object schema into an array of {@link SqlColumn} definitions
|
|
209
|
+
* suitable for SQL table creation.
|
|
210
|
+
*
|
|
211
|
+
* Nested ZodObject fields are recursively flattened into separate columns
|
|
212
|
+
* with underscore-separated names (e.g. `address.street` → `address_street`).
|
|
213
|
+
* Arrays become JSON columns.
|
|
214
|
+
*/
|
|
215
|
+
declare function zodSchemaToColumns(schema: z.ZodObject<any>, dialect: SqlDialect, options?: ZodSchemaToColumnsOptions): SqlColumn[];
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Convert a single Firestore value into a SQL-safe primitive.
|
|
219
|
+
*
|
|
220
|
+
* Complex types (arrays, GeoPoints, binary) become JSON strings.
|
|
221
|
+
* Primitives pass through unchanged.
|
|
222
|
+
* Objects are NOT stringified here — they are flattened by serializeDocument.
|
|
223
|
+
*/
|
|
224
|
+
declare function serializeValue(value: unknown): unknown;
|
|
225
|
+
/**
|
|
226
|
+
* Serialize a full Firestore document into a flat object of SQL-safe values.
|
|
227
|
+
*
|
|
228
|
+
* Nested objects are flattened into underscore-separated column names
|
|
229
|
+
* (e.g. `address.street` → `address_street`). Arrays become JSON strings.
|
|
230
|
+
* Applies optional field exclusions and column renames from `options`.
|
|
231
|
+
*/
|
|
232
|
+
declare function serializeDocument(doc: Record<string, unknown>, options?: Pick<RepoSyncConfig, "exclude" | "columnMap">): Record<string, unknown>;
|
|
233
|
+
|
|
172
234
|
/**
|
|
173
235
|
* Firestore Cloud Function triggers that publish {@link SyncEvent}s to
|
|
174
236
|
* Google Cloud PubSub.
|
|
@@ -211,95 +273,33 @@ declare function autoMigrate<M extends Record<string, any>>(repoMapping: M, adap
|
|
|
211
273
|
declare function createSyncTriggers<M extends Record<string, any>>(repoMapping: M, config: SyncTriggersConfig<NoInfer<M>>): Record<string, any>;
|
|
212
274
|
|
|
213
275
|
/**
|
|
214
|
-
*
|
|
215
|
-
*
|
|
216
|
-
* @
|
|
217
|
-
* ```typescript
|
|
218
|
-
* import * as firestoreTriggers from "firebase-functions/v2/firestore";
|
|
219
|
-
* import * as pubsubHandler from "firebase-functions/v2/pubsub";
|
|
220
|
-
* import { PubSub } from "@google-cloud/pubsub";
|
|
221
|
-
*
|
|
222
|
-
* const sync = createFirestoreSync(repos, {
|
|
223
|
-
* deps: { firestoreTriggers, pubsubHandler, pubsub: new PubSub() },
|
|
224
|
-
* adapter,
|
|
225
|
-
* topicPrefix: "firestore-sync",
|
|
226
|
-
* autoMigrate: true,
|
|
227
|
-
* admin: {
|
|
228
|
-
* auth: { type: "basic", username: "admin", password: "secret" },
|
|
229
|
-
* featuresFlag: { healthCheck: true, manualSync: true, viewQueue: true },
|
|
230
|
-
* },
|
|
231
|
-
* repos: {
|
|
232
|
-
* users: { exclude: ["documentPath"], columnMap: { docId: "user_id" } },
|
|
233
|
-
* posts: { columnMap: { docId: "post_id" } },
|
|
234
|
-
* },
|
|
235
|
-
* });
|
|
236
|
-
*
|
|
237
|
-
* // Triggers + PubSub handlers
|
|
238
|
-
* export const { users_onCreate, users_onUpdate, users_onDelete, sync_users } = sync.functions;
|
|
239
|
-
*
|
|
240
|
-
* // Admin endpoint — wrap with onRequest yourself
|
|
241
|
-
* export const syncAdmin = onRequest(sync.adminHandler!);
|
|
242
|
-
*
|
|
243
|
-
* // Or pass onRequest in admin config to auto-add to sync.functions:
|
|
244
|
-
* // admin: { onRequest, ... } → export const { syncAdmin } = sync.functions;
|
|
245
|
-
* ```
|
|
246
|
-
*/
|
|
247
|
-
|
|
248
|
-
declare function createFirestoreSync<M extends Record<string, any>>(repoMapping: M, config: FirestoreSyncConfig<NoInfer<M>>): {
|
|
249
|
-
/** All Cloud Functions (triggers + handlers + optional admin) — spread into exports */
|
|
250
|
-
functions: Record<string, any>;
|
|
251
|
-
/**
|
|
252
|
-
* Raw admin HTTP handler — wrap with `onRequest()` yourself if you
|
|
253
|
-
* didn't pass `onRequest` in the admin config.
|
|
254
|
-
* @example
|
|
255
|
-
* ```ts
|
|
256
|
-
* export const syncAdmin = onRequest(sync.adminHandler!);
|
|
257
|
-
* ```
|
|
258
|
-
*/
|
|
259
|
-
adminHandler: ((req: any, res: any) => Promise<void>) | null;
|
|
260
|
-
/** Process a SyncEvent directly (for testing) */
|
|
261
|
-
handleMessage: (event: SyncEvent) => Promise<void>;
|
|
262
|
-
/** Internal queue map (for testing) */
|
|
263
|
-
queues: Map<string, SyncQueue>;
|
|
264
|
-
/** Flush all queues and stop timers */
|
|
265
|
-
shutdown: () => Promise<void>;
|
|
266
|
-
};
|
|
267
|
-
|
|
268
|
-
/**
|
|
269
|
-
* Sync Admin — optional HTTP endpoint for inspecting and managing the
|
|
270
|
-
* Firestore → SQL sync pipeline.
|
|
271
|
-
*
|
|
272
|
-
* Features (gated by `featuresFlag`):
|
|
273
|
-
* - **healthCheck** — compare expected Zod-derived columns vs actual SQL columns
|
|
274
|
-
* - **manualSync** — force re-sync all documents in a Firestore collection
|
|
275
|
-
* - **viewQueue** — inspect pending items in the per-repo SyncQueue
|
|
276
|
-
*
|
|
277
|
-
* @example
|
|
278
|
-
* ```typescript
|
|
279
|
-
* const sync = createFirestoreSync(repos, {
|
|
280
|
-
* // …deps, adapter, etc.
|
|
281
|
-
* admin: {
|
|
282
|
-
* auth: { type: "basic", username: "admin", password: "secret" },
|
|
283
|
-
* featuresFlag: { healthCheck: true, manualSync: true, viewQueue: true },
|
|
284
|
-
* },
|
|
285
|
-
* });
|
|
276
|
+
* PubSub worker — creates a Cloud Function that receives {@link SyncEvent}
|
|
277
|
+
* messages from PubSub, routes them to per-repo {@link SyncQueue}s, and
|
|
278
|
+
* flushes batches to the configured {@link SqlAdapter}.
|
|
286
279
|
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
280
|
+
* Dependencies (`firebase-functions`, `@google-cloud/pubsub`) are injected
|
|
281
|
+
* via the `deps` config property.
|
|
289
282
|
*/
|
|
290
283
|
|
|
291
284
|
/**
|
|
292
|
-
* Create
|
|
285
|
+
* Create a PubSub-triggered Cloud Function that syncs Firestore changes
|
|
286
|
+
* to a SQL database.
|
|
293
287
|
*
|
|
294
|
-
*
|
|
295
|
-
*
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
*
|
|
299
|
-
* @param repoConfigs - Per-repo sync config (tableName, exclude, columnMap…)
|
|
300
|
-
* @param pubsub - PubSub client (needed for configCheck)
|
|
301
|
-
* @param topicPrefix - PubSub topic prefix (needed for configCheck)
|
|
288
|
+
* Returns an object with:
|
|
289
|
+
* - `createHandler` — creates a Cloud Function for a PubSub topic
|
|
290
|
+
* - `handleMessage` — process a SyncEvent directly (for testing)
|
|
291
|
+
* - `queues` — internal SyncQueue map (for testing / manual flush)
|
|
292
|
+
* - `shutdown()` — flush all queues and stop timers
|
|
302
293
|
*/
|
|
303
|
-
declare function
|
|
294
|
+
declare function createSyncWorker<M extends Record<string, any>>(repoMapping: M, config: SyncWorkerConfig<NoInfer<M>>): {
|
|
295
|
+
/** Process a SyncEvent directly (for testing or custom PubSub integration). */
|
|
296
|
+
handleMessage: (syncEvent: SyncEvent) => Promise<void>;
|
|
297
|
+
/** Create a Cloud Function handler for a specific PubSub topic. */
|
|
298
|
+
createHandler: (topicName: string) => any;
|
|
299
|
+
/** Internal queue map (for testing). */
|
|
300
|
+
queues: Map<string, SyncQueue>;
|
|
301
|
+
/** Flush all queues and stop timers. */
|
|
302
|
+
shutdown(): Promise<void>;
|
|
303
|
+
};
|
|
304
304
|
|
|
305
|
-
export { FirestoreSyncConfig, GenerateDDLConfig, LogicalType, type MigrateResult, PubSubClientDep, RepoSyncConfig, SqlAdapter, SqlColumn, SqlDialect, SqlTableDef,
|
|
305
|
+
export { FirestoreSyncConfig, GenerateDDLConfig, LogicalType, type MigrateResult, PubSubClientDep, RepoSyncConfig, SqlAdapter, SqlColumn, SqlDialect, SqlTableDef, SyncEvent, SyncQueue, type SyncQueueOptions, SyncTriggersConfig, SyncWorkerConfig, addColumnsDDL, adminsyncConfig, autoMigrate, createFirestoreSync, createSyncTriggers, createSyncWorker, createTableDDL, createadminsyncServer, generateDDL, serializeDocument, serializeValue, zodSchemaToColumns, zodTypeToLogical };
|
package/dist/sync/index.js
CHANGED
|
@@ -1,8 +1,4 @@
|
|
|
1
|
-
var
|
|
2
|
-
|
|
3
|
-
`)}var O=class{constructor(e){this.buffer=[];this.flushing=false;this.timer=null;this.adapter=e.adapter,this.tableName=e.tableName,this.primaryKey=e.primaryKey,this.batchSize=e.batchSize??100,this.onFlushError=e.onFlushError;let t=e.flushIntervalMs??5e3;t>0&&(this.timer=setInterval(()=>{this.flush();},t),typeof this.timer=="object"&&"unref"in this.timer&&this.timer.unref());}get size(){return this.buffer.length}enqueue(...e){this.buffer.push(...e),this.buffer.length>=this.batchSize&&this.flush();}async flush(){if(this.flushing||this.buffer.length===0)return;this.flushing=true;let e=this.buffer.splice(0,this.batchSize);try{let t=[],c=[];for(let r of e)r.operation==="DELETE"?c.push(r.docId):r.data&&t.push(r.data);t.length>0&&await this.adapter.upsertRows(this.tableName,t,this.primaryKey),c.length>0&&await this.adapter.deleteRows(this.tableName,this.primaryKey,c);}catch(t){this.onFlushError?await this.onFlushError(e,t).catch(()=>{}):(this.buffer.unshift(...e),console.error(`[SyncQueue] Flush failed for ${this.tableName}:`,t));}finally{this.flushing=false;}}async shutdown(){this.timer&&(clearInterval(this.timer),this.timer=null),await this.flush();}};var X=new Set;async function se(o,e,t,c,r,l,y){if(X.has(o))return;let v=N(t,e.dialect,{primaryKey:r,exclude:l,columnMap:y});if(!await e.tableExists(c))await e.createTable({tableName:c,columns:v});else {let h=new Set(await e.getTableColumns(c)),g=v.filter(f=>!h.has(f.name));if(g.length>0){let f=e.dialect.addColumnsDDL(c,g);for(let i of f.split(`
|
|
4
|
-
`).filter(Boolean))await e.bigquery?.query?.({query:i})??Promise.resolve();}}X.add(o);}function K(o,e){let{deps:t,adapter:c,batchSize:r=100,flushIntervalMs:l=5e3,autoMigrate:y=false,repos:v={}}=e,s=new Map;function h(i,a){let d=s.get(i);if(d)return d;let p=v[i]?.tableName??i,b=async(u,$)=>{try{let x=t.pubsub.topic(`${i}-sync-dlq`);for(let P of u)await x.publishMessage({json:P});}catch(x){console.error(`[SyncWorker] Dead-letter publish failed for ${i}:`,x);}};return d=new O({adapter:c,tableName:p,primaryKey:a,batchSize:r,flushIntervalMs:l,onFlushError:b}),s.set(i,d),d}async function g(i){let{repoName:a}=i,d=o[a];if(!d){console.warn(`[SyncWorker] Unknown repo "${a}", skipping event`);return}let n=d._systemKeys?.[0]??d.documentKey??"docId";if(y){let b=d.schema??void 0;if(b){let u=v[a],$=u?.tableName??a;await se(a,c,b,$,n,u?.exclude,u?.columnMap);}}h(a,n).enqueue(i);}function f(i){return t.pubsubHandler.onMessagePublished(i,async a=>{let d=a.data?.message?.json??a.data?.json;if(!d){console.warn("[SyncWorker] Received empty PubSub message");return}await g(d);})}return {handleMessage:g,createHandler:f,queues:s,async shutdown(){let i=[];for(let a of s.values())i.push(a.shutdown());await Promise.all(i);}}}async function ae(o,e,t){let c={created:[],altered:[],upToDate:[],skipped:[]};for(let[r,l]of Object.entries(o)){let y=l.schema??void 0;if(!y){c.skipped.push(r);continue}let v=t?.repos?.[r],s=v?.tableName??r,h=l._systemKeys?.[0]??l.documentKey??"docId",g=N(y,e.dialect,{primaryKey:h,exclude:v?.exclude,columnMap:v?.columnMap}),f={tableName:s,columns:g};if(!await e.tableExists(s))await e.createTable(f),c.created.push(s);else {let a=new Set(await e.getTableColumns(s)),d=g.filter(n=>!a.has(n.name));if(d.length>0){let n=e.dialect.addColumnsDDL(s,d);for(let p of n.split(`
|
|
5
|
-
`).filter(b=>b.trim()))await ie(e,p);c.altered.push(s);}else c.upToDate.push(s);}}return c}async function ie(o,e){let t=o;typeof t.executeRaw=="function"?await t.executeRaw(e):typeof t.bigquery?.query=="function"?await t.bigquery.query({query:e}):console.warn("[autoMigrate] Adapter does not support raw SQL execution; skipping:",e);}var ce="firestore-sync";function de(o,e){let t=e.ref?.path??void 0;return t?`${t}/{docId}`:(console.warn(`[SyncTriggers] Cannot determine collection path for "${o}". Skipping.`),null)}function Q(o,e){let{onDocumentCreated:t,onDocumentUpdated:c,onDocumentDeleted:r}=e.deps.firestoreTriggers,l=e.deps.pubsub,y=e?.topicPrefix??ce,v={};for(let[s,h]of Object.entries(o)){let g=e?.repos?.[s],f;if(h._isGroup){if(!g?.triggerPath){console.warn(`[SyncTriggers] Skipping collection-group repo "${s}". Provide a triggerPath in the sync repos config for group collections.`);continue}f=g.triggerPath;}else f=g?.triggerPath??de(s,h);if(!f)continue;let i=h._systemKeys?.[0]??"docId",a=`${y}-${s}`;v[`${s}_onCreate`]=t(f,async d=>{let n=d.data;if(!n)return;let p=n.data();if(!p)return;let b=String(p[i]??n.id),u=_(p,{exclude:g?.exclude,columnMap:g?.columnMap}),$={operation:"INSERT",repoName:s,docId:b,data:u,timestamp:new Date().toISOString()};await l.topic(a).publishMessage({json:$});}),v[`${s}_onUpdate`]=c(f,async d=>{let n=d.data?.after;if(!n)return;let p=n.data();if(!p)return;let b=String(p[i]??n.id),u=_(p,{exclude:g?.exclude,columnMap:g?.columnMap}),$={operation:"UPSERT",repoName:s,docId:b,data:u,timestamp:new Date().toISOString()};await l.topic(a).publishMessage({json:$});}),v[`${s}_onDelete`]=r(f,async d=>{let n=d.data;if(!n)return;let p=n.data(),b=String(p?.[i]??n.id),u={operation:"DELETE",repoName:s,docId:b,data:null,timestamp:new Date().toISOString()};await l.topic(a).publishMessage({json:u});});}return v}function ue(o){let e=[],t=o.replace(/[.*+?^${}()|[\]\\]/g,c=>c===":"?c:`\\${c}`).replace(/:([a-zA-Z_][a-zA-Z0-9_]*)/g,(c,r)=>(e.push(r),"([^/]+)"));return {pattern:new RegExp(`^${t}$`),paramNames:e}}function le(o){let e=o.path??o.url??"/",t=e.indexOf("?");return t===-1?e:e.slice(0,t)}var M=class{constructor(){this.routes=[];this.middlewares=[];this.notFoundHandler=(e,t)=>{t.status(404).send("Not Found");};this.errorHandler=(e,t,c)=>{console.error("[MiniRouter]",e),c.status(500).send("Internal Server Error");};}use(e){return this.middlewares.push(e),this}get(e,t){return this.addRoute("GET",e,t)}post(e,t){return this.addRoute("POST",e,t)}put(e,t){return this.addRoute("PUT",e,t)}patch(e,t){return this.addRoute("PATCH",e,t)}delete(e,t){return this.addRoute("DELETE",e,t)}onNotFound(e){return this.notFoundHandler=e,this}onError(e){return this.errorHandler=e,this}addRoute(e,t,c){let{pattern:r,paramNames:l}=ue(t);return this.routes.push({method:e.toUpperCase(),pattern:r,paramNames:l,handler:c}),this}async handle(e,t){let c=(e.method??"GET").toUpperCase(),r=le(e),l=null,y={};for(let h of this.routes){if(h.method!==c)continue;let g=r.match(h.pattern);if(g){l=h,y={},h.paramNames.forEach((f,i)=>{y[f]=decodeURIComponent(g[i+1]??"");});break}}let v=Object.assign(e,{params:y}),s=l?l.handler:this.notFoundHandler;try{await this.runMiddlewareChain(v,t,s);}catch(h){this.errorHandler(h,e,t);}}async runMiddlewareChain(e,t,c){let r=0,l=async()=>{if(r<this.middlewares.length){let y=this.middlewares[r++];await y(e,t,l);}else await c(e,t);};await l();}};function E(o,e){if(process.env.FUNCTIONS_EMULATOR==="true"){let r=process.env.GCLOUD_PROJECT??process.env.GOOGLE_CLOUD_PROJECT??"demo-project",l=process.env.FUNCTION_REGION??"us-central1",y=(process.env.FUNCTION_TARGET??"").replace(/\./g,"-");return `/${r}/${l}/${y}${e}`}let t=process.env.K_SERVICE,c=o.hostname??o.headers?.host??"";return t&&c.includes("cloudfunctions.net")?`/${t}${e}`:e}function D(o,e,t){return `<!DOCTYPE html>
|
|
1
|
+
function te(o){let e=[],t=o.replace(/[.*+?^${}()|[\]\\]/g,c=>c===":"?c:`\\${c}`).replace(/:([a-zA-Z_][a-zA-Z0-9_]*)/g,(c,r)=>(e.push(r),"([^/]+)"));return {pattern:new RegExp(`^${t}$`),paramNames:e}}function ne(o){let e=o.path??o.url??"/",t=e.indexOf("?");return t===-1?e:e.slice(0,t)}var I=class{constructor(){this.routes=[];this.middlewares=[];this.notFoundHandler=(e,t)=>{t.status(404).send("Not Found");};this.errorHandler=(e,t,c)=>{console.error("[MiniRouter]",e),c.status(500).send("Internal Server Error");};}use(e){return this.middlewares.push(e),this}get(e,t){return this.addRoute("GET",e,t)}post(e,t){return this.addRoute("POST",e,t)}put(e,t){return this.addRoute("PUT",e,t)}patch(e,t){return this.addRoute("PATCH",e,t)}delete(e,t){return this.addRoute("DELETE",e,t)}onNotFound(e){return this.notFoundHandler=e,this}onError(e){return this.errorHandler=e,this}addRoute(e,t,c){let{pattern:r,paramNames:p}=te(t);return this.routes.push({method:e.toUpperCase(),pattern:r,paramNames:p,handler:c}),this}async handle(e,t){let c=(e.method??"GET").toUpperCase(),r=ne(e),p=null,y={};for(let h of this.routes){if(h.method!==c)continue;let f=r.match(h.pattern);if(f){p=h,y={},h.paramNames.forEach((b,i)=>{y[b]=decodeURIComponent(f[i+1]??"");});break}}let $=Object.assign(e,{params:y}),a=p?p.handler:this.notFoundHandler;try{await this.runMiddlewareChain($,t,a);}catch(h){this.errorHandler(h,e,t);}}async runMiddlewareChain(e,t,c){let r=0,p=async()=>{if(r<this.middlewares.length){let y=this.middlewares[r++];await y(e,t,p);}else await c(e,t);};await p();}};var oe={string:"ZodString",number:"ZodNumber",bigint:"ZodBigInt",boolean:"ZodBoolean",date:"ZodDate",enum:"ZodEnum",nativeEnum:"ZodNativeEnum",literal:"ZodLiteral",object:"ZodObject",array:"ZodArray",optional:"ZodOptional",nullable:"ZodNullable",default:"ZodDefault",coerce:"ZodCoerce",union:"ZodUnion",undefined:"ZodUndefined",unknown:"ZodUnknown",any:"ZodAny",record:"ZodRecord"};function M(o){let e=o,t=e._zod?.def?.type;if(t)return oe[t]??`Zod${t.charAt(0).toUpperCase()}${t.slice(1)}`;let c=e._def?.typeName;return c||""}function B(o){let e=o;if(e._zod?.def?.innerType)return e._zod.def.innerType;if(e._def?.innerType)return e._def.innerType}function F(o){let e=o;return e.shape&&typeof e.shape=="object"?e.shape:e._zod?.def?.shape&&typeof e._zod.def.shape=="object"?e._zod.def.shape:e._def?.shape?typeof e._def.shape=="function"?e._def.shape():e._def.shape:{}}var re=new Set(["ZodOptional","ZodNullable","ZodDefault"]);function G(o){let e=o,t=false;for(;;){let c=M(e);if(!re.has(c))break;(c==="ZodOptional"||c==="ZodNullable")&&(t=true);let r=B(e);if(!r)break;e=r;}return {inner:e,nullable:t}}var W={ZodString:"string",ZodNumber:"number",ZodBigInt:"bigint",ZodBoolean:"boolean",ZodDate:"timestamp",ZodEnum:"string",ZodNativeEnum:"string",ZodLiteral:"string"};function se(o){let{inner:e}=G(o);return W[M(e)]??"json"}function J(o,e,t,c,r,p,y,$){for(let[a,h]of Object.entries(o)){let f=t?`${t}__${a}`:a;if(r.has(a)||r.has(f))continue;let{inner:b,nullable:i}=G(h),s=M(b),d=c||i;if(s==="ZodObject"){let u=F(b);J(u,e,f,d,r,p,y,$);continue}let n=W[s]??"json",l=f===y||a===y,g=p[f]??p[a]??f;$.push({name:g,sqlType:e.mapType(n),nullable:l?false:d,isPrimaryKey:l});}}function A(o,e,t={}){let{primaryKey:c,exclude:r=[],columnMap:p={}}=t,y=new Set(r),$=F(o),a=[];return J($,e,"",false,y,p,c,a),a}function U(o){if(o==null)return null;if(typeof o=="object"&&typeof o.toDate=="function")return o.toDate().toISOString();if(o instanceof Date)return o.toISOString();if(Buffer.isBuffer(o))return o.toString("base64");if(o instanceof Uint8Array)return Buffer.from(o).toString("base64");if(typeof o=="object"&&"latitude"in o&&"longitude"in o){let e=o;return JSON.stringify({lat:e.latitude,lng:e.longitude})}return Array.isArray(o)?JSON.stringify(o.map(U)):o}function V(o,e,t){for(let[c,r]of Object.entries(o)){let p=e?`${e}__${c}`:c;r!=null&&typeof r=="object"&&!Array.isArray(r)&&!(r instanceof Date)&&!Buffer.isBuffer(r)&&!(r instanceof Uint8Array)&&typeof r.toDate!="function"&&!("latitude"in r&&"longitude"in r)?V(r,p,t):t[p]=U(r);}}function _(o,e){let t=new Set(e?.exclude),c=e?.columnMap??{},r={};V(o,"",r);let p={};for(let[y,$]of Object.entries(r)){if(t.has(y))continue;let a=y.split("__")[0];if(a!==y&&t.has(a))continue;let h=c[y]??y;p[h]=$;}return p}function E(o,e){if(process.env.FUNCTIONS_EMULATOR==="true"){let r=process.env.GCLOUD_PROJECT??process.env.GOOGLE_CLOUD_PROJECT??"demo-project",p=process.env.FUNCTION_REGION??"us-central1",y=(process.env.FUNCTION_TARGET??"").replace(/\./g,"-");return `/${r}/${p}/${y}${e}`}let t=process.env.K_SERVICE,c=o.hostname??o.headers?.host??"";return t&&c.includes("cloudfunctions.net")?`/${t.toLowerCase()}${e}`:e}function D(o,e,t){return `<!DOCTYPE html>
|
|
6
2
|
<html lang="en"><head>
|
|
7
3
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
8
4
|
<title>${o} \u2014 Sync Admin</title>
|
|
@@ -32,61 +28,65 @@ var ee={string:"ZodString",number:"ZodNumber",bigint:"ZodBigInt",boolean:"ZodBoo
|
|
|
32
28
|
<nav><a href="${e}/">\u2190 Dashboard</a></nav>
|
|
33
29
|
<h1>${o}</h1>
|
|
34
30
|
${t}
|
|
35
|
-
</body></html>`}function
|
|
36
|
-
<td><strong>${
|
|
37
|
-
<td>${
|
|
38
|
-
<td>${
|
|
39
|
-
<td>${
|
|
40
|
-
<td>${
|
|
31
|
+
</body></html>`}function N(o,e,t=200){o.status(t).set("Content-Type","text/html; charset=utf-8").send(e);}function q(o,e,t=200){o.status(t).set("Content-Type","application/json").send(JSON.stringify(e,null,2));}function z(o){return (o.headers?.accept??"").includes("application/json")}function K(o,e,t,c,r,p,y,$){let a=(r.basePath??"/").replace(/\/$/,"")||"",h=r.featuresFlag??{},f=[];for(let[i,s]of Object.entries(o)){let d=p[i];f.push({name:i,schema:s.schema??null,documentKey:s._systemKeys?.[0]??s.documentKey??"docId",tableName:d?.tableName??i,isGroup:!!s._isGroup,repoCfg:d,repo:s});}let b=new I;if(r.auth)if(typeof r.auth=="function")b.use(r.auth);else {let i=r.auth.realm??"Sync Admin",s="Basic "+Buffer.from(`${r.auth.username}:${r.auth.password}`).toString("base64");b.use((d,n,l)=>{if((d.headers?.authorization??"")!==s){n.status(401).set("WWW-Authenticate",`Basic realm="${i}"`).set("Content-Type","text/plain").send("Unauthorized");return}l();});}return b.get(`${a}/`,(i,s)=>{let d=E(i,a),n=f.map(v=>{let C=[];return h.healthCheck&&C.push(`<a class="btn" href="${d}/${v.name}/health">Health</a>`),h.manualSync&&C.push(`<a class="btn btn-primary" href="${d}/${v.name}/force-sync">Force Sync</a>`),`<tr>
|
|
32
|
+
<td><strong>${v.name}</strong></td>
|
|
33
|
+
<td>${v.tableName}</td>
|
|
34
|
+
<td>${v.isGroup?'<span class="badge badge-warn">group</span>':'<span class="badge badge-ok">collection</span>'}</td>
|
|
35
|
+
<td>${v.schema?"\u2713":"\u2717"}</td>
|
|
36
|
+
<td>${C.join(" ")}</td>
|
|
41
37
|
</tr>`}).join(`
|
|
42
|
-
`),
|
|
38
|
+
`),l=h.viewQueue?`<p><a class="btn" href="${d}/queues">View Queues</a></p>`:"",g=h.configCheck?`<p style="margin-top:.5rem"><a class="btn" href="${d}/config-check">\u2699 Config Check</a></p>`:"",u=D("Sync Dashboard",d,`<div class="card">
|
|
43
39
|
<table>
|
|
44
40
|
<thead><tr><th>Repository</th><th>Table</th><th>Type</th><th>Schema</th><th>Actions</th></tr></thead>
|
|
45
41
|
<tbody>${n}</tbody>
|
|
46
42
|
</table>
|
|
47
|
-
${
|
|
48
|
-
${
|
|
49
|
-
</div>`);
|
|
43
|
+
${l}
|
|
44
|
+
${g}
|
|
45
|
+
</div>`);N(s,u);}),b.get(`${a}`,(i,s)=>{let d=E(i,a);s.status(302).set("Location",`${d}/`).send("");}),h.healthCheck&&b.get(`${a}/:repoName/health`,async(i,s)=>{let d=E(i,a),n=f.find(m=>m.name===i.params.repoName);if(!n){N(s,D("Not Found",d,`<p>Unknown repo: ${i.params.repoName}</p>`),404);return}if(!n.schema){N(s,D("Health Check",d,`<p class="badge badge-warn">No Zod schema attached to "${n.name}"</p>`));return}let l=A(n.schema,e.dialect,{primaryKey:n.documentKey,exclude:n.repoCfg?.exclude,columnMap:n.repoCfg?.columnMap}),g=[],u=false,v=null;try{u=await e.tableExists(n.tableName),u&&(g=await e.getTableColumns(n.tableName));}catch(m){v=m?.message??String(m);}let C=new Set(g),P=new Set(l.map(m=>m.name)),O=l.filter(m=>!C.has(m.name)),T=g.filter(m=>!P.has(m)),j=l.filter(m=>C.has(m.name)),S=u&&O.length===0&&!v;if(z(i)){q(s,{repo:n.name,table:n.tableName,tableExists:u,healthy:S,error:v,columns:{expected:l.map(m=>({name:m.name,type:m.sqlType,nullable:m.nullable,isPrimaryKey:m.isPrimaryKey})),actual:g,matched:j.map(m=>m.name),missing:O.map(m=>({name:m.name,type:m.sqlType})),extra:T}});return}let R=S?'<span class="badge badge-ok">Healthy</span>':'<span class="badge badge-err">Unhealthy</span>',x=l.map(m=>{let L=C.has(m.name)?'<span class="badge badge-ok">OK</span>':'<span class="badge badge-err">MISSING</span>';return `<tr><td>${m.name}</td><td>${m.sqlType}</td><td>${m.nullable?"Yes":"No"}</td><td>${m.isPrimaryKey?"\u2713":""}</td><td>${L}</td></tr>`}).join(`
|
|
50
46
|
`),w=T.map(m=>`<tr><td>${m}</td><td colspan="3" class="muted">not in schema</td><td><span class="badge badge-warn">EXTRA</span></td></tr>`).join(`
|
|
51
47
|
`),k=D(`Health: ${n.name}`,d,`<div class="card">
|
|
52
48
|
<p>Table: <code>${n.tableName}</code> ${u?R:'<span class="badge badge-err">NOT FOUND</span>'}</p>
|
|
53
|
-
${
|
|
49
|
+
${v?`<p class="badge badge-err">Error: ${v}</p>`:""}
|
|
54
50
|
<h2>Columns</h2>
|
|
55
51
|
<table>
|
|
56
52
|
<thead><tr><th>Column</th><th>SQL Type</th><th>Nullable</th><th>PK</th><th>Status</th></tr></thead>
|
|
57
|
-
<tbody>${
|
|
53
|
+
<tbody>${x}${w}</tbody>
|
|
58
54
|
</table>
|
|
59
|
-
</div>`);
|
|
55
|
+
</div>`);N(s,k);}),h.manualSync&&(b.get(`${a}/:repoName/force-sync`,(i,s)=>{let d=E(i,a),n=f.find(g=>g.name===i.params.repoName);if(!n){N(s,D("Not Found",d,`<p>Unknown repo: ${i.params.repoName}</p>`),404);return}let l=D(`Force Sync: ${n.name}`,d,`<div class="card">
|
|
60
56
|
<p>This will read <strong>all</strong> documents from the <code>${n.name}</code> Firestore collection
|
|
61
57
|
and upsert them into the <code>${n.tableName}</code> SQL table.</p>
|
|
62
58
|
<p class="muted" style="margin:.75rem 0">This may take a while for large collections.</p>
|
|
63
59
|
<form method="POST" action="${d}/${n.name}/force-sync">
|
|
64
60
|
<button type="submit" class="btn btn-primary">Start Force Sync</button>
|
|
65
61
|
</form>
|
|
66
|
-
</div>`);
|
|
62
|
+
</div>`);N(s,l);}),b.post(`${a}/:repoName/force-sync`,async(i,s)=>{let d=E(i,a),n=f.find(T=>T.name===i.params.repoName);if(!n){q(s,{error:`Unknown repo: ${i.params.repoName}`},404);return}let l=n.repo.ref;if(!l){q(s,{error:`No collection reference for "${n.name}"`},400);return}let g=0,u=0,v=500,C=l.limit(v),P=null;try{for(;;){let S=await(P?C.startAfter(P):C).get();if(S.empty)break;for(let R of S.docs){let x=R.data(),w=String(x[n.documentKey]??R.id),k=_(x,{exclude:n.repoCfg?.exclude,columnMap:n.repoCfg?.columnMap});try{await c({operation:"UPSERT",repoName:n.name,docId:w,data:k,timestamp:new Date().toISOString()}),g++;}catch{u++;}}if(P=S.docs[S.docs.length-1],S.docs.length<v)break}let T=t.get(n.name);T&&await T.flush();}catch(T){if(z(i)){q(s,{error:T?.message??String(T),synced:g,errors:u},500);return}N(s,D(`Force Sync: ${n.name}`,d,`<div class="card">
|
|
67
63
|
<p class="badge badge-err">Error: ${T?.message??String(T)}</p>
|
|
68
|
-
<p>Synced ${
|
|
69
|
-
</div>`),500);return}if(
|
|
64
|
+
<p>Synced ${g} docs before failure (${u} errors).</p>
|
|
65
|
+
</div>`),500);return}if(z(i)){q(s,{repo:n.name,table:n.tableName,synced:g,errors:u});return}let O=D(`Force Sync: ${n.name}`,d,`<div class="card">
|
|
70
66
|
<p class="badge badge-ok">Complete</p>
|
|
71
|
-
<p>Synced <strong>${
|
|
67
|
+
<p>Synced <strong>${g}</strong> documents to <code>${n.tableName}</code>.</p>
|
|
72
68
|
${u>0?`<p class="badge badge-warn">${u} error(s)</p>`:""}
|
|
73
|
-
</div>`);
|
|
74
|
-
`),
|
|
69
|
+
</div>`);N(s,O);})),h.viewQueue&&b.get(`${a}/queues`,(i,s)=>{let d=E(i,a),n=[];for(let u of f){let v=t.get(u.name);n.push({repo:u.name,table:u.tableName,pending:v?v.size:0});}if(z(i)){q(s,{queues:n});return}let l=n.map(u=>`<tr><td>${u.repo}</td><td>${u.table}</td><td>${u.pending===0?'<span class="badge badge-ok">0</span>':`<span class="badge badge-warn">${u.pending}</span>`}</td></tr>`).join(`
|
|
70
|
+
`),g=D("Sync Queues",d,`<div class="card">
|
|
75
71
|
<table>
|
|
76
72
|
<thead><tr><th>Repository</th><th>Table</th><th>Pending</th></tr></thead>
|
|
77
|
-
<tbody>${
|
|
73
|
+
<tbody>${l}</tbody>
|
|
78
74
|
</table>
|
|
79
|
-
</div>`);
|
|
80
|
-
`),console:`${
|
|
81
|
-
<td>${
|
|
75
|
+
</div>`);N(s,g);}),h.configCheck&&b.get(`${a}/config-check`,async(i,s)=>{let d=E(i,a),n=process.env.GCLOUD_PROJECT??process.env.GOOGLE_CLOUD_PROJECT??process.env.GCP_PROJECT??"unknown",l="https://console.cloud.google.com",g=$??"firestore-sync",u=[];try{await e.tableExists("__nonexistent_health_check__"),u.push({name:"BigQuery API",category:"bigquery",status:"ok",message:"BigQuery API is reachable"});}catch(S){let R=S?.message??String(S),x=R.toLowerCase(),w=x.includes("disabled")||x.includes("has not been used")||x.includes("accessnotconfigured"),k=x.includes("permission")||R.includes("403")||x.includes("access denied"),m=x.includes("project")&&x.includes("not found"),L=x.includes("not found")||R.includes("404");w?u.push({name:"BigQuery API",category:"bigquery",status:"error",message:"BigQuery API is not enabled",fix:{gcloud:`gcloud services enable bigquery.googleapis.com --project=${n}`,console:`${l}/apis/library/bigquery.googleapis.com?project=${n}`}}):m?u.push({name:"BigQuery Project",category:"bigquery",status:"error",message:R,fix:{hint:"The GCP project does not exist or the credentials don't have access to it. In the Firebase emulator, GCLOUD_PROJECT may override the configured projectId. Ensure you pass the correct projectId to the BigQuery constructor and have valid credentials.",console:`${l}/home/dashboard`}}):k?u.push({name:"BigQuery API",category:"bigquery",status:"error",message:`Permission denied: ${R}`,fix:{hint:"Grant the service account BigQuery Data Editor + BigQuery Job User roles",gcloud:[`SA=$(gcloud run services describe YOUR_SERVICE --region=YOUR_REGION --format="value(spec.template.spec.serviceAccountName)" --project=${n})`,`gcloud projects add-iam-policy-binding ${n} --member="serviceAccount:$SA" --role="roles/bigquery.dataEditor"`,`gcloud projects add-iam-policy-binding ${n} --member="serviceAccount:$SA" --role="roles/bigquery.jobUser"`].join(`
|
|
76
|
+
`),console:`${l}/iam-admin/iam?project=${n}`}}):L?u.push({name:"BigQuery Dataset",category:"bigquery",status:"error",message:`Dataset not found: ${R}`,fix:{hint:"Create the dataset first",gcloud:`bq mk --dataset ${n}:YOUR_DATASET_ID`,console:`${l}/bigquery?project=${n}`}}):u.push({name:"BigQuery API",category:"bigquery",status:"ok",message:"BigQuery API is reachable (table lookup returned expected error)"});}for(let S of f)try{let R=await e.tableExists(S.tableName);u.push({name:`Table: ${S.tableName}`,category:"bigquery",status:R?"ok":"warn",message:R?`Table \`${S.tableName}\` exists`:`Table \`${S.tableName}\` does not exist yet`,...!R&&{fix:{hint:"Table will be auto-created on first sync if autoMigrate is enabled. Or create it manually."}}});}catch(R){u.push({name:`Table: ${S.tableName}`,category:"bigquery",status:"error",message:R?.message??String(R)});}if(y)for(let S of f){let R=`${g}-${S.name}`;try{let x=y.topic(R);if(typeof x.exists=="function"){let[w]=await x.exists();u.push({name:`Topic: ${R}`,category:"pubsub",status:w?"ok":"error",message:w?`Topic \`${R}\` exists`:`Topic \`${R}\` does not exist`,...!w&&{fix:{gcloud:`gcloud pubsub topics create ${R} --project=${n}`,console:`${l}/cloudpubsub/topic/list?project=${n}`}}});}else u.push({name:`Topic: ${R}`,category:"pubsub",status:"warn",message:"Cannot verify topic existence (PubSub client doesn't expose .exists())",fix:{gcloud:`gcloud pubsub topics create ${R} --project=${n}`,console:`${l}/cloudpubsub/topic/list?project=${n}`,hint:"Ensure the topic exists. It is auto-created by the Firebase emulator but must exist in production."}});}catch(x){let w=x?.message??String(x),k=w.includes("disabled")||w.includes("has not been used");if(u.push({name:k?"Pub/Sub API":`Topic: ${R}`,category:"pubsub",status:"error",message:k?"Pub/Sub API is not enabled":w,fix:k?{gcloud:`gcloud services enable pubsub.googleapis.com --project=${n}`,console:`${l}/apis/library/pubsub.googleapis.com?project=${n}`}:{gcloud:`gcloud pubsub topics create ${R} --project=${n}`,console:`${l}/cloudpubsub/topic/list?project=${n}`}}),k)break}}else u.push({name:"Pub/Sub Client",category:"pubsub",status:"warn",message:"PubSub client not available for config check"});if(z(i)){let S=u.every(R=>R.status==="ok");q(s,{project:n,healthy:S,checks:u});return}let v=S=>S==="ok"?'<span class="badge badge-ok">OK</span>':S==="warn"?'<span class="badge badge-warn">WARN</span>':'<span class="badge badge-err">ERROR</span>',C={bigquery:u.filter(S=>S.category==="bigquery"),pubsub:u.filter(S=>S.category==="pubsub"),firestore:u.filter(S=>S.category==="firestore")},P=(S,R)=>{if(R.length===0)return "";let x=R.map(w=>{let k="";if(w.fix){let m=[];w.fix.hint&&m.push(`<p class="muted">${w.fix.hint}</p>`),w.fix.gcloud&&m.push(`<pre>$ ${w.fix.gcloud}</pre>`),w.fix.console&&m.push(`<p><a href="${w.fix.console}" target="_blank">Open GCP Console \u2192</a></p>`),k=`<div style="margin-top:.5rem">${m.join("")}</div>`;}return `<tr>
|
|
77
|
+
<td>${v(w.status)}</td>
|
|
82
78
|
<td><strong>${w.name}</strong><br><span class="muted">${w.message}</span>${k}</td>
|
|
83
79
|
</tr>`}).join(`
|
|
84
80
|
`);return `<h2>${S}</h2>
|
|
85
81
|
<table><thead><tr><th style="width:80px">Status</th><th>Check</th></tr></thead>
|
|
86
|
-
<tbody>${
|
|
82
|
+
<tbody>${x}</tbody></table>`},T=u.every(S=>S.status==="ok")?'<span class="badge badge-ok">All checks passed</span>':'<span class="badge badge-warn">Some issues found</span>',j=D("Config Check",d,`<div class="card">
|
|
87
83
|
<p>Project: <code>${n}</code> ${T}</p>
|
|
88
|
-
${P("BigQuery",
|
|
89
|
-
${P("Pub/Sub",
|
|
90
|
-
${P("Firestore",
|
|
91
|
-
</div>`);
|
|
84
|
+
${P("BigQuery",C.bigquery)}
|
|
85
|
+
${P("Pub/Sub",C.pubsub)}
|
|
86
|
+
${P("Firestore",C.firestore)}
|
|
87
|
+
</div>`);N(s,j);}),async(i,s)=>{await b.handle(i,s);}}var ae="firestore-sync";function ie(o,e){let t=e.ref?.path??void 0;return t?`${t}/{docId}`:(console.warn(`[SyncTriggers] Cannot determine collection path for "${o}". Skipping.`),null)}function Q(o,e){let{onDocumentCreated:t,onDocumentUpdated:c,onDocumentDeleted:r}=e.deps.firestoreTriggers,p=e.deps.pubsub,y=e?.topicPrefix??ae,$={};for(let[a,h]of Object.entries(o)){let f=e?.repos?.[a],b;if(h._isGroup){if(!f?.triggerPath){console.warn(`[SyncTriggers] Skipping collection-group repo "${a}". Provide a triggerPath in the sync repos config for group collections.`);continue}b=f.triggerPath;}else b=f?.triggerPath??ie(a,h);if(!b)continue;let i=h._systemKeys?.[0]??"docId",s=`${y}-${a}`;$[`${a}_onCreate`]=t(b,async d=>{let n=d.data;if(!n)return;let l=n.data();if(!l)return;let g=String(l[i]??n.id),u=_(l,{exclude:f?.exclude,columnMap:f?.columnMap}),v={operation:"INSERT",repoName:a,docId:g,data:u,timestamp:new Date().toISOString()};await p.topic(s).publishMessage({json:v});}),$[`${a}_onUpdate`]=c(b,async d=>{let n=d.data?.after;if(!n)return;let l=n.data();if(!l)return;let g=String(l[i]??n.id),u=_(l,{exclude:f?.exclude,columnMap:f?.columnMap}),v={operation:"UPSERT",repoName:a,docId:g,data:u,timestamp:new Date().toISOString()};await p.topic(s).publishMessage({json:v});}),$[`${a}_onDelete`]=r(b,async d=>{let n=d.data;if(!n)return;let l=n.data(),g=String(l?.[i]??n.id),u={operation:"DELETE",repoName:a,docId:g,data:null,timestamp:new Date().toISOString()};await p.topic(s).publishMessage({json:u});});}return $}var Z=class{constructor(e){this.buffer=[];this.flushing=false;this.timer=null;this.adapter=e.adapter,this.tableName=e.tableName,this.primaryKey=e.primaryKey,this.batchSize=e.batchSize??100,this.onFlushError=e.onFlushError;let t=e.flushIntervalMs??5e3;t>0&&(this.timer=setInterval(()=>{this.flush();},t),typeof this.timer=="object"&&"unref"in this.timer&&this.timer.unref());}get size(){return this.buffer.length}enqueue(...e){this.buffer.push(...e),this.buffer.length>=this.batchSize&&this.flush();}async flush(){if(this.flushing||this.buffer.length===0)return;this.flushing=true;let e=this.buffer.splice(0,this.batchSize);try{let t=[],c=[];for(let r of e)r.operation==="DELETE"?c.push(r.docId):r.data&&t.push(r.data);t.length>0&&await this.adapter.upsertRows(this.tableName,t,this.primaryKey),c.length>0&&await this.adapter.deleteRows(this.tableName,this.primaryKey,c);}catch(t){this.onFlushError?await this.onFlushError(e,t).catch(()=>{}):(this.buffer.unshift(...e),console.error(`[SyncQueue] Flush failed for ${this.tableName}:`,t));}finally{this.flushing=false;}}async shutdown(){this.timer&&(clearInterval(this.timer),this.timer=null),await this.flush();}};var Y=new Set;async function ce(o,e,t,c,r,p,y){if(Y.has(o))return;let $=A(t,e.dialect,{primaryKey:r,exclude:p,columnMap:y});if(!await e.tableExists(c))await e.createTable({tableName:c,columns:$});else {let h=new Set(await e.getTableColumns(c)),f=$.filter(b=>!h.has(b.name));if(f.length>0){let b=e.dialect.addColumnsDDL(c,f);for(let i of b.split(`
|
|
88
|
+
`).filter(Boolean))await e.bigquery?.query?.({query:i})??Promise.resolve();}}Y.add(o);}function H(o,e){let{deps:t,adapter:c,batchSize:r=100,flushIntervalMs:p=5e3,autoMigrate:y=false,repos:$={}}=e,a=new Map;function h(i,s){let d=a.get(i);if(d)return d;let l=$[i]?.tableName??i,g=async(u,v)=>{try{let C=t.pubsub.topic(`${i}-sync-dlq`);for(let P of u)await C.publishMessage({json:P});}catch(C){console.error(`[SyncWorker] Dead-letter publish failed for ${i}:`,C);}};return d=new Z({adapter:c,tableName:l,primaryKey:s,batchSize:r,flushIntervalMs:p,onFlushError:g}),a.set(i,d),d}async function f(i){let{repoName:s}=i,d=o[s];if(!d){console.warn(`[SyncWorker] Unknown repo "${s}", skipping event`);return}let n=d._systemKeys?.[0]??d.documentKey??"docId";if(y){let g=d.schema??void 0;if(g){let u=$[s],v=u?.tableName??s;await ce(s,c,g,v,n,u?.exclude,u?.columnMap);}}h(s,n).enqueue(i);}function b(i){return t.pubsubHandler.onMessagePublished(i,async s=>{let d=s.data?.message?.json??s.data?.json;if(!d){console.warn("[SyncWorker] Received empty PubSub message");return}await f(d);})}return {handleMessage:f,createHandler:b,queues:a,async shutdown(){let i=[];for(let s of a.values())i.push(s.shutdown());await Promise.all(i);}}}var de="firestore-sync";function X(o){if(typeof o!="function")return o;let e=o,t;return new Proxy({},{get(c,r){return t||(t=e()),t[r]},has(c,r){return t||(t=e()),r in t}})}function ue(o,e){let{deps:t,adapter:c,topicPrefix:r=de,batchSize:p,flushIntervalMs:y,autoMigrate:$,admin:a,repos:h}=e,f=X(t.pubsub),b=X(c),i=Q(o,{deps:{firestoreTriggers:t.firestoreTriggers,pubsub:f},topicPrefix:r,repos:h}),s=H(o,{deps:{pubsubHandler:t.pubsubHandler,pubsub:f},adapter:b,batchSize:p,flushIntervalMs:y,autoMigrate:$,repos:h}),d={};for(let g of Object.keys(o))d[`sync_${g}`]=s.createHandler(`${r}-${g}`);let n=null;a&&(n=K(o,b,s.queues,s.handleMessage,a,h??{},f,r),d.adminsync=a.onRequest?a.httpsOptions?a.onRequest(a.httpsOptions,n):a.onRequest(n):n);let l={functions:{...i,...d},adminHandler:n,handleMessage:s.handleMessage,queues:s.queues,shutdown:s.shutdown};for(let g of ["adminHandler","handleMessage","queues","shutdown"])Object.defineProperty(l,g,{enumerable:false});return l}function ee(o,e){return o.createTableDDL(e)}function le(o,e,t){return o.addColumnsDDL(e,t)}function pe(o,e,t){let c=[];for(let[r,p]of Object.entries(o)){let y=p.schema??p._schema??void 0;if(!y)continue;let $=t?.repos?.[r],a=$?.tableName??r,h=p._systemKeys?.[0]??p.documentKey??"docId",f=A(y,e,{primaryKey:h,exclude:$?.exclude,columnMap:$?.columnMap}),b={tableName:a,columns:f};c.push(ee(e,b));}return c.join(`
|
|
89
|
+
|
|
90
|
+
`)}async function fe(o,e,t){let c={created:[],altered:[],upToDate:[],skipped:[]};for(let[r,p]of Object.entries(o)){let y=p.schema??void 0;if(!y){c.skipped.push(r);continue}let $=t?.repos?.[r],a=$?.tableName??r,h=p._systemKeys?.[0]??p.documentKey??"docId",f=A(y,e.dialect,{primaryKey:h,exclude:$?.exclude,columnMap:$?.columnMap}),b={tableName:a,columns:f};if(!await e.tableExists(a))await e.createTable(b),c.created.push(a);else {let s=new Set(await e.getTableColumns(a)),d=f.filter(n=>!s.has(n.name));if(d.length>0){let n=e.dialect.addColumnsDDL(a,d);for(let l of n.split(`
|
|
91
|
+
`).filter(g=>g.trim()))await ge(e,l);c.altered.push(a);}else c.upToDate.push(a);}}return c}async function ge(o,e){let t=o;typeof t.executeRaw=="function"?await t.executeRaw(e):typeof t.bigquery?.query=="function"?await t.bigquery.query({query:e}):console.warn("[autoMigrate] Adapter does not support raw SQL execution; skipping:",e);}export{Z as SyncQueue,le as addColumnsDDL,fe as autoMigrate,ue as createFirestoreSync,Q as createSyncTriggers,H as createSyncWorker,ee as createTableDDL,K as createadminsyncServer,pe as generateDDL,_ as serializeDocument,U as serializeValue,A as zodSchemaToColumns,se as zodTypeToLogical};//# sourceMappingURL=index.js.map
|
|
92
92
|
//# sourceMappingURL=index.js.map
|