@lpdjs/firestore-repo-service 2.1.13 → 2.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/sync/bigquery.d.cts +1 -1
- package/dist/sync/bigquery.d.ts +1 -1
- package/dist/sync/index.cjs +69 -40
- package/dist/sync/index.cjs.map +1 -1
- package/dist/sync/index.d.cts +81 -3
- package/dist/sync/index.d.ts +81 -3
- package/dist/sync/index.js +69 -40
- package/dist/sync/index.js.map +1 -1
- package/dist/{types-Cy9G-Lx2.d.cts → types-CcgJAKfP.d.cts} +37 -0
- package/dist/{types-Cy9G-Lx2.d.ts → types-CcgJAKfP.d.ts} +37 -0
- package/package.json +1 -1
package/dist/sync/index.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { S as SqlAdapter, a as SyncEvent, b as adminsyncConfig, R as RepoSyncConfig, P as PubSubClientDep, F as FirestoreSyncConfig, c as SqlDialect, d as SqlColumn, e as SqlTableDef, G as GenerateDDLConfig, L as LogicalType, f as SyncTriggersConfig, g as SyncWorkerConfig } from '../types-
|
|
2
|
-
export { h as FirestoreTriggersDep, O as OrFactory, i as PubSubHandlerDep, j as SyncDeps, k as SyncOperation, l as adminsyncBasicAuth, m as adminsyncFeaturesFlag } from '../types-
|
|
1
|
+
import { S as SqlAdapter, a as SyncEvent, b as adminsyncConfig, R as RepoSyncConfig, P as PubSubClientDep, F as FirestoreSyncConfig, c as SqlDialect, d as SqlColumn, e as SqlTableDef, G as GenerateDDLConfig, L as LogicalType, f as SyncTriggersConfig, g as SyncWorkerConfig } from '../types-CcgJAKfP.cjs';
|
|
2
|
+
export { h as FirestoreTriggersDep, O as OrFactory, i as PubSubHandlerDep, j as SyncDeps, k as SyncOperation, l as adminsyncBasicAuth, m as adminsyncFeaturesFlag } from '../types-CcgJAKfP.cjs';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -239,6 +239,84 @@ declare function serializeValue(value: unknown): unknown;
|
|
|
239
239
|
*/
|
|
240
240
|
declare function serializeDocument(doc: Record<string, unknown>, options?: Pick<RepoSyncConfig, "exclude" | "columnMap">): Record<string, unknown>;
|
|
241
241
|
|
|
242
|
+
/**
|
|
243
|
+
* Pub/Sub infrastructure setup for the Firestore → SQL sync.
|
|
244
|
+
*
|
|
245
|
+
* Cloud Functions v2 (`onMessagePublished`) auto-creates topics and push
|
|
246
|
+
* subscriptions on deploy — **but without `enableMessageOrdering`**. This
|
|
247
|
+
* helper pre-creates the topics and subscriptions with ordering enabled, so
|
|
248
|
+
* that the Cloud Function reuses the existing subscription on deploy.
|
|
249
|
+
*
|
|
250
|
+
* Run it as a one-off script (e.g. in CI before `firebase deploy`) or call
|
|
251
|
+
* it manually from a setup script. Idempotent: existing resources are kept
|
|
252
|
+
* as-is (subscriptions are NOT recreated, since `enableMessageOrdering` is
|
|
253
|
+
* immutable after creation — a warning is logged if mismatched).
|
|
254
|
+
*
|
|
255
|
+
* @example
|
|
256
|
+
* ```typescript
|
|
257
|
+
* import { PubSub } from "@google-cloud/pubsub";
|
|
258
|
+
* import { ensureSyncInfra } from "@lpdjs/firestore-repo-service/sync";
|
|
259
|
+
*
|
|
260
|
+
* await ensureSyncInfra(repos, {
|
|
261
|
+
* pubsub: new PubSub(),
|
|
262
|
+
* topicPrefix: "firestore-sync",
|
|
263
|
+
* ordering: true,
|
|
264
|
+
* subscriptionSuffix: "sync-sub", // default
|
|
265
|
+
* includeDLQ: true,
|
|
266
|
+
* });
|
|
267
|
+
* ```
|
|
268
|
+
*/
|
|
269
|
+
|
|
270
|
+
interface EnsureSyncInfraOptions {
|
|
271
|
+
/** PubSub client (`new PubSub()` from `@google-cloud/pubsub`). */
|
|
272
|
+
pubsub: PubSubClientDep;
|
|
273
|
+
/** Topic prefix — must match the value used by `createSyncTriggers`. */
|
|
274
|
+
topicPrefix?: string;
|
|
275
|
+
/**
|
|
276
|
+
* Whether to enable message ordering on the created subscriptions.
|
|
277
|
+
* Default: `true`.
|
|
278
|
+
*/
|
|
279
|
+
ordering?: boolean;
|
|
280
|
+
/**
|
|
281
|
+
* Suffix appended to each topic name to derive the subscription name.
|
|
282
|
+
* Final name: `{topicPrefix}-{repoName}-{subscriptionSuffix}`.
|
|
283
|
+
* Default: `"sync-sub"`.
|
|
284
|
+
*/
|
|
285
|
+
subscriptionSuffix?: string;
|
|
286
|
+
/**
|
|
287
|
+
* Also create the dead-letter topic (`{prefix}-{repoName}-dlq`) used by
|
|
288
|
+
* `createSyncWorker` on flush failures. Default: `true`.
|
|
289
|
+
*/
|
|
290
|
+
includeDLQ?: boolean;
|
|
291
|
+
/**
|
|
292
|
+
* Ack deadline in seconds for the created subscriptions. Default: `60`.
|
|
293
|
+
*/
|
|
294
|
+
ackDeadlineSeconds?: number;
|
|
295
|
+
/**
|
|
296
|
+
* Optional message retention duration (e.g. `"604800s"` for 7 days).
|
|
297
|
+
*/
|
|
298
|
+
messageRetentionDuration?: string;
|
|
299
|
+
}
|
|
300
|
+
interface EnsureSyncInfraResult {
|
|
301
|
+
topics: {
|
|
302
|
+
name: string;
|
|
303
|
+
created: boolean;
|
|
304
|
+
}[];
|
|
305
|
+
subscriptions: {
|
|
306
|
+
name: string;
|
|
307
|
+
topic: string;
|
|
308
|
+
created: boolean;
|
|
309
|
+
orderingEnabled: boolean;
|
|
310
|
+
/** Set when an existing subscription has the wrong ordering setting. */
|
|
311
|
+
warning?: string;
|
|
312
|
+
}[];
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Idempotently create the Pub/Sub topics + subscriptions used by the sync
|
|
316
|
+
* pipeline, with `enableMessageOrdering` set on subscriptions.
|
|
317
|
+
*/
|
|
318
|
+
declare function ensureSyncInfra<M extends Record<string, any>>(repoMapping: M, opts: EnsureSyncInfraOptions): Promise<EnsureSyncInfraResult>;
|
|
319
|
+
|
|
242
320
|
/**
|
|
243
321
|
* Firestore Cloud Function triggers that publish {@link SyncEvent}s to
|
|
244
322
|
* Google Cloud PubSub.
|
|
@@ -310,4 +388,4 @@ declare function createSyncWorker<M extends Record<string, any>>(repoMapping: M,
|
|
|
310
388
|
shutdown(): Promise<void>;
|
|
311
389
|
};
|
|
312
390
|
|
|
313
|
-
export { FirestoreSyncConfig, GenerateDDLConfig, LogicalType, type MigrateResult, PubSubClientDep, RepoSyncConfig, SqlAdapter, SqlColumn, SqlDialect, SqlTableDef, SyncEvent, SyncQueue, type SyncQueueOptions, SyncTriggersConfig, SyncWorkerConfig, addColumnsDDL, adminsyncConfig, autoMigrate, createFirestoreSync, createSyncTriggers, createSyncWorker, createTableDDL, createadminsyncServer, generateDDL, serializeDocument, serializeValue, zodSchemaToColumns, zodTypeToLogical };
|
|
391
|
+
export { type EnsureSyncInfraOptions, type EnsureSyncInfraResult, FirestoreSyncConfig, GenerateDDLConfig, LogicalType, type MigrateResult, PubSubClientDep, RepoSyncConfig, SqlAdapter, SqlColumn, SqlDialect, SqlTableDef, SyncEvent, SyncQueue, type SyncQueueOptions, SyncTriggersConfig, SyncWorkerConfig, addColumnsDDL, adminsyncConfig, autoMigrate, createFirestoreSync, createSyncTriggers, createSyncWorker, createTableDDL, createadminsyncServer, ensureSyncInfra, generateDDL, serializeDocument, serializeValue, zodSchemaToColumns, zodTypeToLogical };
|
package/dist/sync/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { S as SqlAdapter, a as SyncEvent, b as adminsyncConfig, R as RepoSyncConfig, P as PubSubClientDep, F as FirestoreSyncConfig, c as SqlDialect, d as SqlColumn, e as SqlTableDef, G as GenerateDDLConfig, L as LogicalType, f as SyncTriggersConfig, g as SyncWorkerConfig } from '../types-
|
|
2
|
-
export { h as FirestoreTriggersDep, O as OrFactory, i as PubSubHandlerDep, j as SyncDeps, k as SyncOperation, l as adminsyncBasicAuth, m as adminsyncFeaturesFlag } from '../types-
|
|
1
|
+
import { S as SqlAdapter, a as SyncEvent, b as adminsyncConfig, R as RepoSyncConfig, P as PubSubClientDep, F as FirestoreSyncConfig, c as SqlDialect, d as SqlColumn, e as SqlTableDef, G as GenerateDDLConfig, L as LogicalType, f as SyncTriggersConfig, g as SyncWorkerConfig } from '../types-CcgJAKfP.js';
|
|
2
|
+
export { h as FirestoreTriggersDep, O as OrFactory, i as PubSubHandlerDep, j as SyncDeps, k as SyncOperation, l as adminsyncBasicAuth, m as adminsyncFeaturesFlag } from '../types-CcgJAKfP.js';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -239,6 +239,84 @@ declare function serializeValue(value: unknown): unknown;
|
|
|
239
239
|
*/
|
|
240
240
|
declare function serializeDocument(doc: Record<string, unknown>, options?: Pick<RepoSyncConfig, "exclude" | "columnMap">): Record<string, unknown>;
|
|
241
241
|
|
|
242
|
+
/**
|
|
243
|
+
* Pub/Sub infrastructure setup for the Firestore → SQL sync.
|
|
244
|
+
*
|
|
245
|
+
* Cloud Functions v2 (`onMessagePublished`) auto-creates topics and push
|
|
246
|
+
* subscriptions on deploy — **but without `enableMessageOrdering`**. This
|
|
247
|
+
* helper pre-creates the topics and subscriptions with ordering enabled, so
|
|
248
|
+
* that the Cloud Function reuses the existing subscription on deploy.
|
|
249
|
+
*
|
|
250
|
+
* Run it as a one-off script (e.g. in CI before `firebase deploy`) or call
|
|
251
|
+
* it manually from a setup script. Idempotent: existing resources are kept
|
|
252
|
+
* as-is (subscriptions are NOT recreated, since `enableMessageOrdering` is
|
|
253
|
+
* immutable after creation — a warning is logged if mismatched).
|
|
254
|
+
*
|
|
255
|
+
* @example
|
|
256
|
+
* ```typescript
|
|
257
|
+
* import { PubSub } from "@google-cloud/pubsub";
|
|
258
|
+
* import { ensureSyncInfra } from "@lpdjs/firestore-repo-service/sync";
|
|
259
|
+
*
|
|
260
|
+
* await ensureSyncInfra(repos, {
|
|
261
|
+
* pubsub: new PubSub(),
|
|
262
|
+
* topicPrefix: "firestore-sync",
|
|
263
|
+
* ordering: true,
|
|
264
|
+
* subscriptionSuffix: "sync-sub", // default
|
|
265
|
+
* includeDLQ: true,
|
|
266
|
+
* });
|
|
267
|
+
* ```
|
|
268
|
+
*/
|
|
269
|
+
|
|
270
|
+
interface EnsureSyncInfraOptions {
|
|
271
|
+
/** PubSub client (`new PubSub()` from `@google-cloud/pubsub`). */
|
|
272
|
+
pubsub: PubSubClientDep;
|
|
273
|
+
/** Topic prefix — must match the value used by `createSyncTriggers`. */
|
|
274
|
+
topicPrefix?: string;
|
|
275
|
+
/**
|
|
276
|
+
* Whether to enable message ordering on the created subscriptions.
|
|
277
|
+
* Default: `true`.
|
|
278
|
+
*/
|
|
279
|
+
ordering?: boolean;
|
|
280
|
+
/**
|
|
281
|
+
* Suffix appended to each topic name to derive the subscription name.
|
|
282
|
+
* Final name: `{topicPrefix}-{repoName}-{subscriptionSuffix}`.
|
|
283
|
+
* Default: `"sync-sub"`.
|
|
284
|
+
*/
|
|
285
|
+
subscriptionSuffix?: string;
|
|
286
|
+
/**
|
|
287
|
+
* Also create the dead-letter topic (`{prefix}-{repoName}-dlq`) used by
|
|
288
|
+
* `createSyncWorker` on flush failures. Default: `true`.
|
|
289
|
+
*/
|
|
290
|
+
includeDLQ?: boolean;
|
|
291
|
+
/**
|
|
292
|
+
* Ack deadline in seconds for the created subscriptions. Default: `60`.
|
|
293
|
+
*/
|
|
294
|
+
ackDeadlineSeconds?: number;
|
|
295
|
+
/**
|
|
296
|
+
* Optional message retention duration (e.g. `"604800s"` for 7 days).
|
|
297
|
+
*/
|
|
298
|
+
messageRetentionDuration?: string;
|
|
299
|
+
}
|
|
300
|
+
interface EnsureSyncInfraResult {
|
|
301
|
+
topics: {
|
|
302
|
+
name: string;
|
|
303
|
+
created: boolean;
|
|
304
|
+
}[];
|
|
305
|
+
subscriptions: {
|
|
306
|
+
name: string;
|
|
307
|
+
topic: string;
|
|
308
|
+
created: boolean;
|
|
309
|
+
orderingEnabled: boolean;
|
|
310
|
+
/** Set when an existing subscription has the wrong ordering setting. */
|
|
311
|
+
warning?: string;
|
|
312
|
+
}[];
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Idempotently create the Pub/Sub topics + subscriptions used by the sync
|
|
316
|
+
* pipeline, with `enableMessageOrdering` set on subscriptions.
|
|
317
|
+
*/
|
|
318
|
+
declare function ensureSyncInfra<M extends Record<string, any>>(repoMapping: M, opts: EnsureSyncInfraOptions): Promise<EnsureSyncInfraResult>;
|
|
319
|
+
|
|
242
320
|
/**
|
|
243
321
|
* Firestore Cloud Function triggers that publish {@link SyncEvent}s to
|
|
244
322
|
* Google Cloud PubSub.
|
|
@@ -310,4 +388,4 @@ declare function createSyncWorker<M extends Record<string, any>>(repoMapping: M,
|
|
|
310
388
|
shutdown(): Promise<void>;
|
|
311
389
|
};
|
|
312
390
|
|
|
313
|
-
export { FirestoreSyncConfig, GenerateDDLConfig, LogicalType, type MigrateResult, PubSubClientDep, RepoSyncConfig, SqlAdapter, SqlColumn, SqlDialect, SqlTableDef, SyncEvent, SyncQueue, type SyncQueueOptions, SyncTriggersConfig, SyncWorkerConfig, addColumnsDDL, adminsyncConfig, autoMigrate, createFirestoreSync, createSyncTriggers, createSyncWorker, createTableDDL, createadminsyncServer, generateDDL, serializeDocument, serializeValue, zodSchemaToColumns, zodTypeToLogical };
|
|
391
|
+
export { type EnsureSyncInfraOptions, type EnsureSyncInfraResult, FirestoreSyncConfig, GenerateDDLConfig, LogicalType, type MigrateResult, PubSubClientDep, RepoSyncConfig, SqlAdapter, SqlColumn, SqlDialect, SqlTableDef, SyncEvent, SyncQueue, type SyncQueueOptions, SyncTriggersConfig, SyncWorkerConfig, addColumnsDDL, adminsyncConfig, autoMigrate, createFirestoreSync, createSyncTriggers, createSyncWorker, createTableDDL, createadminsyncServer, ensureSyncInfra, generateDDL, serializeDocument, serializeValue, zodSchemaToColumns, zodTypeToLogical };
|
package/dist/sync/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
function
|
|
1
|
+
function oe(o){let e=[],n=o.replace(/[.*+?^${}()|[\]\\]/g,i=>i===":"?i:`\\${i}`).replace(/:([a-zA-Z_][a-zA-Z0-9_]*)/g,(i,r)=>(e.push(r),"([^/]+)"));return {pattern:new RegExp(`^${n}$`),paramNames:e}}function re(o){let e=o.path??o.url??"/",n=e.indexOf("?");return n===-1?e:e.slice(0,n)}var M=class{constructor(){this.routes=[];this.middlewares=[];this.notFoundHandler=(e,n)=>{n.status(404).send("Not Found");};this.errorHandler=(e,n,i)=>{console.error("[MiniRouter]",e),i.status(500).send("Internal Server Error");};}use(e){return this.middlewares.push(e),this}get(e,n){return this.addRoute("GET",e,n)}post(e,n){return this.addRoute("POST",e,n)}put(e,n){return this.addRoute("PUT",e,n)}patch(e,n){return this.addRoute("PATCH",e,n)}delete(e,n){return this.addRoute("DELETE",e,n)}onNotFound(e){return this.notFoundHandler=e,this}onError(e){return this.errorHandler=e,this}addRoute(e,n,i){let{pattern:r,paramNames:g}=oe(n);return this.routes.push({method:e.toUpperCase(),pattern:r,paramNames:g,handler:i}),this}async handle(e,n){let i=(e.method??"GET").toUpperCase(),r=re(e),g=null,m={};for(let y of this.routes){if(y.method!==i)continue;let b=r.match(y.pattern);if(b){g=y,m={},y.paramNames.forEach((S,l)=>{m[S]=decodeURIComponent(b[l+1]??"");});break}}let $=Object.assign(e,{params:m}),u=g?g.handler:this.notFoundHandler;try{await this.runMiddlewareChain($,n,u);}catch(y){this.errorHandler(y,e,n);}}async runMiddlewareChain(e,n,i){let r=0,g=async()=>{if(r<this.middlewares.length){let m=this.middlewares[r++];await m(e,n,g);}else await i(e,n);};await g();}};var se={string:"ZodString",number:"ZodNumber",bigint:"ZodBigInt",boolean:"ZodBoolean",date:"ZodDate",enum:"ZodEnum",nativeEnum:"ZodNativeEnum",literal:"ZodLiteral",object:"ZodObject",array:"ZodArray",optional:"ZodOptional",nullable:"ZodNullable",default:"ZodDefault",coerce:"ZodCoerce",union:"ZodUnion",undefined:"ZodUndefined",unknown:"ZodUnknown",any:"ZodAny",record:"ZodRecord"};function L(o){let e=o,n=e._zod?.def?.type;if(n)return se[n]??`Zod${n.charAt(0).toUpperCase()}${n.slice(1)}`;let i=e._def?.typeName;return i||""}function G(o){let e=o;if(e._zod?.def?.innerType)return e._zod.def.innerType;if(e._def?.innerType)return e._def.innerType}function F(o){let e=o;return e.shape&&typeof e.shape=="object"?e.shape:e._zod?.def?.shape&&typeof e._zod.def.shape=="object"?e._zod.def.shape:e._def?.shape?typeof e._def.shape=="function"?e._def.shape():e._def.shape:{}}var ae=new Set(["ZodOptional","ZodNullable","ZodDefault"]);function W(o){let e=o,n=false;for(;;){let i=L(e);if(!ae.has(i))break;(i==="ZodOptional"||i==="ZodNullable")&&(n=true);let r=G(e);if(!r)break;e=r;}return {inner:e,nullable:n}}var J={ZodString:"string",ZodNumber:"number",ZodBigInt:"bigint",ZodBoolean:"boolean",ZodDate:"timestamp",ZodEnum:"string",ZodNativeEnum:"string",ZodLiteral:"string"};function ie(o){let{inner:e}=W(o);return J[L(e)]??"json"}function V(o,e,n,i,r,g,m,$){for(let[u,y]of Object.entries(o)){let b=n?`${n}__${u}`:u;if(r.has(u)||r.has(b))continue;let{inner:S,nullable:l}=W(y),a=L(S),s=i||l;if(a==="ZodObject"){let c=F(S);V(c,e,b,s,r,g,m,$);continue}let t=J[a]??"json",d=b===m||u===m,p=g[b]??g[u]??b;$.push({name:p,sqlType:e.mapType(t),nullable:d?false:s,isPrimaryKey:d});}}function A(o,e,n={}){let{primaryKey:i,exclude:r=[],columnMap:g={}}=n,m=new Set(r),$=F(o),u=[];return V($,e,"",false,m,g,i,u),u}function K(o){if(o==null)return null;if(typeof o=="object"&&typeof o.toDate=="function")return o.toDate().toISOString();if(o instanceof Date)return o.toISOString();if(Buffer.isBuffer(o))return o.toString("base64");if(o instanceof Uint8Array)return Buffer.from(o).toString("base64");if(typeof o=="object"&&"latitude"in o&&"longitude"in o){let e=o;return JSON.stringify({lat:e.latitude,lng:e.longitude})}return Array.isArray(o)?JSON.stringify(o.map(K)):o}function Y(o,e,n){for(let[i,r]of Object.entries(o)){let g=e?`${e}__${i}`:i;r!=null&&typeof r=="object"&&!Array.isArray(r)&&!(r instanceof Date)&&!Buffer.isBuffer(r)&&!(r instanceof Uint8Array)&&typeof r.toDate!="function"&&!("latitude"in r&&"longitude"in r)?Y(r,g,n):n[g]=K(r);}}function z(o,e){let n=new Set(e?.exclude),i=e?.columnMap??{},r={};Y(o,"",r);let g={};for(let[m,$]of Object.entries(r)){if(n.has(m))continue;let u=m.split("__")[0];if(u!==m&&n.has(u))continue;let y=i[m]??(m.includes("__")?i[m.split("__").pop()]:void 0)??m;g[y]=$;}return g}async function Q(o,e){let{pubsub:n,topicPrefix:i="firestore-sync",ordering:r=true,subscriptionSuffix:g="sync-sub",includeDLQ:m=true,ackDeadlineSeconds:$=60,messageRetentionDuration:u}=e,y={topics:[],subscriptions:[]};for(let b of Object.keys(o)){let S=`${i}-${b}`,l=`${i}-${b}-${g}`,a=n.topic(S),s=false,[t]=await a.exists();t||(await a.create(),s=true,console.info(`[ensureSyncInfra] Created topic "${S}"`)),y.topics.push({name:S,created:s});let d=a.subscription(l),[p]=await d.exists();if(!p)await d.create({enableMessageOrdering:r,ackDeadlineSeconds:$,...u?{messageRetentionDuration:u}:{}}),console.info(`[ensureSyncInfra] Created subscription "${l}" (ordering=${r})`),y.subscriptions.push({name:l,topic:S,created:true,orderingEnabled:r});else {let c,f=r;try{let[C]=await d.getMetadata();f=!!C?.enableMessageOrdering,f!==r&&(c=`Subscription "${l}" exists with enableMessageOrdering=${f}, but ordering=${r} was requested. This setting is immutable; delete and recreate the subscription to change it.`,console.warn(`[ensureSyncInfra] ${c}`));}catch{}y.subscriptions.push({name:l,topic:S,created:false,orderingEnabled:f,...c?{warning:c}:{}});}if(m){let c=`${i}-${b}-dlq`,f=n.topic(c),[C]=await f.exists(),w=false;C||(await f.create(),w=true,console.info(`[ensureSyncInfra] Created DLQ topic "${c}"`)),y.topics.push({name:c,created:w});}}return y}function I(o,e){if(process.env.FUNCTIONS_EMULATOR==="true"){let r=process.env.GCLOUD_PROJECT??process.env.GOOGLE_CLOUD_PROJECT??"demo-project",g=process.env.FUNCTION_REGION??"us-central1",m=(process.env.FUNCTION_TARGET??"").replace(/\./g,"-");return `/${r}/${g}/${m}${e}`}let n=process.env.K_SERVICE,i=o.hostname??o.headers?.host??"";return n&&i.includes("cloudfunctions.net")?`/${n.toLowerCase()}${e}`:e}function N(o,e,n){return `<!DOCTYPE html>
|
|
2
2
|
<html lang="en"><head>
|
|
3
3
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
4
4
|
<title>${o} \u2014 Sync Admin</title>
|
|
@@ -28,67 +28,96 @@ function te(o){let e=[],n=o.replace(/[.*+?^${}()|[\]\\]/g,s=>s===":"?s:`\\${s}`)
|
|
|
28
28
|
<nav><a href="${e}/">\u2190 Dashboard</a></nav>
|
|
29
29
|
<h1>${o}</h1>
|
|
30
30
|
${n}
|
|
31
|
-
</body></html>`}function
|
|
32
|
-
<td><strong>${
|
|
33
|
-
<td>${
|
|
34
|
-
<td>${
|
|
35
|
-
<td>${
|
|
31
|
+
</body></html>`}function O(o,e,n=200){o.status(n).set("Content-Type","text/html; charset=utf-8").send(e);}function q(o,e,n=200){o.status(n).set("Content-Type","application/json").send(JSON.stringify(e,null,2));}function j(o){return (o.headers?.accept??"").includes("application/json")}function U(o,e,n,i,r,g,m,$){let u=(r.basePath??"/").replace(/\/$/,"")||"",y=r.featuresFlag??{},b=[];for(let[l,a]of Object.entries(o)){let s=g[l];b.push({name:l,schema:a.schema??null,documentKey:a._systemKeys?.[0]??a.documentKey??"docId",tableName:s?.tableName??l,isGroup:!!a._isGroup,repoCfg:s,repo:a});}let S=new M;if(r.auth)if(typeof r.auth=="function")S.use(r.auth);else {let l=r.auth.realm??"Sync Admin",a="Basic "+Buffer.from(`${r.auth.username}:${r.auth.password}`).toString("base64");S.use((s,t,d)=>{if((s.headers?.authorization??"")!==a){t.status(401).set("WWW-Authenticate",`Basic realm="${l}"`).set("Content-Type","text/plain").send("Unauthorized");return}d();});}return S.get(`${u}/`,(l,a)=>{let s=I(l,u),t=b.map(f=>{let C=[];return y.healthCheck&&C.push(`<a class="btn" href="${s}/${f.name}/health">Health</a>`),y.manualSync&&C.push(`<a class="btn btn-primary" href="${s}/${f.name}/force-sync">Force Sync</a>`),`<tr>
|
|
32
|
+
<td><strong>${f.name}</strong></td>
|
|
33
|
+
<td>${f.tableName}</td>
|
|
34
|
+
<td>${f.isGroup?'<span class="badge badge-warn">group</span>':'<span class="badge badge-ok">collection</span>'}</td>
|
|
35
|
+
<td>${f.schema?"\u2713":"\u2717"}</td>
|
|
36
36
|
<td>${C.join(" ")}</td>
|
|
37
37
|
</tr>`}).join(`
|
|
38
|
-
`),
|
|
38
|
+
`),d=y.viewQueue?`<p><a class="btn" href="${s}/queues">View Queues</a></p>`:"",p=y.configCheck?`<p style="margin-top:.5rem"><a class="btn" href="${s}/config-check">\u2699 Config Check</a></p>`:"",c=N("Sync Dashboard",s,`<div class="card">
|
|
39
39
|
<table>
|
|
40
40
|
<thead><tr><th>Repository</th><th>Table</th><th>Type</th><th>Schema</th><th>Actions</th></tr></thead>
|
|
41
41
|
<tbody>${t}</tbody>
|
|
42
42
|
</table>
|
|
43
|
-
${
|
|
44
|
-
${
|
|
45
|
-
</div>`);
|
|
46
|
-
`),
|
|
47
|
-
`),
|
|
48
|
-
<p>Table: <code>${t.tableName}</code> ${
|
|
49
|
-
${
|
|
43
|
+
${d}
|
|
44
|
+
${p}
|
|
45
|
+
</div>`);O(a,c);}),S.get(`${u}`,(l,a)=>{let s=I(l,u);a.status(302).set("Location",`${s}/`).send("");}),y.healthCheck&&S.get(`${u}/:repoName/health`,async(l,a)=>{let s=I(l,u),t=b.find(h=>h.name===l.params.repoName);if(!t){O(a,N("Not Found",s,`<p>Unknown repo: ${l.params.repoName}</p>`),404);return}if(!t.schema){O(a,N("Health Check",s,`<p class="badge badge-warn">No Zod schema attached to "${t.name}"</p>`));return}let d=A(t.schema,e.dialect,{primaryKey:t.documentKey,exclude:t.repoCfg?.exclude,columnMap:t.repoCfg?.columnMap}),p=[],c=false,f=null;try{c=await e.tableExists(t.tableName),c&&(p=await e.getTableColumns(t.tableName));}catch(h){f=h?.message??String(h);}let C=new Set(p),w=new Set(d.map(h=>h.name)),k=d.filter(h=>!C.has(h.name)),T=p.filter(h=>!w.has(h)),D=d.filter(h=>C.has(h.name)),E=c&&k.length===0&&!f;if(j(l)){q(a,{repo:t.name,table:t.tableName,tableExists:c,healthy:E,error:f,columns:{expected:d.map(h=>({name:h.name,type:h.sqlType,nullable:h.nullable,isPrimaryKey:h.isPrimaryKey})),actual:p,matched:D.map(h=>h.name),missing:k.map(h=>({name:h.name,type:h.sqlType})),extra:T}});return}let x=E?'<span class="badge badge-ok">Healthy</span>':'<span class="badge badge-err">Unhealthy</span>',R=d.map(h=>{let _=C.has(h.name)?'<span class="badge badge-ok">OK</span>':'<span class="badge badge-err">MISSING</span>';return `<tr><td>${h.name}</td><td>${h.sqlType}</td><td>${h.nullable?"Yes":"No"}</td><td>${h.isPrimaryKey?"\u2713":""}</td><td>${_}</td></tr>`}).join(`
|
|
46
|
+
`),P=T.map(h=>`<tr><td>${h}</td><td colspan="3" class="muted">not in schema</td><td><span class="badge badge-warn">EXTRA</span></td></tr>`).join(`
|
|
47
|
+
`),v=N(`Health: ${t.name}`,s,`<div class="card">
|
|
48
|
+
<p>Table: <code>${t.tableName}</code> ${c?x:'<span class="badge badge-err">NOT FOUND</span>'}</p>
|
|
49
|
+
${f?`<p class="badge badge-err">Error: ${f}</p>`:""}
|
|
50
50
|
<h2>Columns</h2>
|
|
51
51
|
<table>
|
|
52
52
|
<thead><tr><th>Column</th><th>SQL Type</th><th>Nullable</th><th>PK</th><th>Status</th></tr></thead>
|
|
53
|
-
<tbody>${
|
|
53
|
+
<tbody>${R}${P}</tbody>
|
|
54
54
|
</table>
|
|
55
|
-
</div>`);
|
|
55
|
+
</div>`);O(a,v);}),y.manualSync&&(S.get(`${u}/:repoName/force-sync`,(l,a)=>{let s=I(l,u),t=b.find(p=>p.name===l.params.repoName);if(!t){O(a,N("Not Found",s,`<p>Unknown repo: ${l.params.repoName}</p>`),404);return}let d=N(`Force Sync: ${t.name}`,s,`<div class="card">
|
|
56
56
|
<p>This will read <strong>all</strong> documents from the <code>${t.name}</code> Firestore collection
|
|
57
57
|
and upsert them into the <code>${t.tableName}</code> SQL table.</p>
|
|
58
58
|
<p class="muted" style="margin:.75rem 0">This may take a while for large collections.</p>
|
|
59
|
-
<form method="POST" action="${
|
|
59
|
+
<form method="POST" action="${s}/${t.name}/force-sync">
|
|
60
60
|
<button type="submit" class="btn btn-primary">Start Force Sync</button>
|
|
61
61
|
</form>
|
|
62
|
-
</div>`);
|
|
63
|
-
<p class="badge badge-err">Error: ${
|
|
64
|
-
<p>Synced ${
|
|
65
|
-
</div>`),500);return}if(j(l)){
|
|
62
|
+
</div>`);O(a,d);}),S.post(`${u}/:repoName/force-sync`,async(l,a)=>{let s=I(l,u),t=b.find(T=>T.name===l.params.repoName);if(!t){q(a,{error:`Unknown repo: ${l.params.repoName}`},404);return}let d=t.repo.ref;if(!d){q(a,{error:`No collection reference for "${t.name}"`},400);return}let p=0,c=0,f=500,C=d.limit(f),w=null;try{for(;;){let E=await(w?C.startAfter(w):C).get();if(E.empty)break;for(let x of E.docs){let R=x.data(),P=String(R[t.documentKey]??x.id),v=z(R,{exclude:t.repoCfg?.exclude,columnMap:t.repoCfg?.columnMap});try{await i({operation:"UPSERT",repoName:t.name,docId:P,data:v,timestamp:new Date().toISOString()}),p++;}catch{c++;}}if(w=E.docs[E.docs.length-1],E.docs.length<f)break}let T=n.get(t.name);T&&await T.flush();}catch(T){if(j(l)){q(a,{error:T?.message??String(T),synced:p,errors:c},500);return}O(a,N(`Force Sync: ${t.name}`,s,`<div class="card">
|
|
63
|
+
<p class="badge badge-err">Error: ${T?.message??String(T)}</p>
|
|
64
|
+
<p>Synced ${p} docs before failure (${c} errors).</p>
|
|
65
|
+
</div>`),500);return}if(j(l)){q(a,{repo:t.name,table:t.tableName,synced:p,errors:c});return}let k=N(`Force Sync: ${t.name}`,s,`<div class="card">
|
|
66
66
|
<p class="badge badge-ok">Complete</p>
|
|
67
|
-
<p>Synced <strong>${
|
|
68
|
-
${
|
|
69
|
-
</div>`);
|
|
70
|
-
`),
|
|
67
|
+
<p>Synced <strong>${p}</strong> documents to <code>${t.tableName}</code>.</p>
|
|
68
|
+
${c>0?`<p class="badge badge-warn">${c} error(s)</p>`:""}
|
|
69
|
+
</div>`);O(a,k);})),y.viewQueue&&S.get(`${u}/queues`,(l,a)=>{let s=I(l,u),t=[];for(let c of b){let f=n.get(c.name);t.push({repo:c.name,table:c.tableName,pending:f?f.size:0});}if(j(l)){q(a,{queues:t});return}let d=t.map(c=>`<tr><td>${c.repo}</td><td>${c.table}</td><td>${c.pending===0?'<span class="badge badge-ok">0</span>':`<span class="badge badge-warn">${c.pending}</span>`}</td></tr>`).join(`
|
|
70
|
+
`),p=N("Sync Queues",s,`<div class="card">
|
|
71
71
|
<table>
|
|
72
72
|
<thead><tr><th>Repository</th><th>Table</th><th>Pending</th></tr></thead>
|
|
73
|
-
<tbody>${
|
|
73
|
+
<tbody>${d}</tbody>
|
|
74
74
|
</table>
|
|
75
|
-
</div>`);
|
|
76
|
-
`),console:`${
|
|
77
|
-
<td>${
|
|
78
|
-
<td><strong>${v.name}</strong><br><span class="muted">${v.message}</span>${
|
|
75
|
+
</div>`);O(a,p);}),y.configCheck&&(S.get(`${u}/config-check`,async(l,a)=>{let s=I(l,u),t=process.env.GCLOUD_PROJECT??process.env.GOOGLE_CLOUD_PROJECT??process.env.GCP_PROJECT??"unknown",d="https://console.cloud.google.com",p=$??"firestore-sync",c=[];try{await e.tableExists("__nonexistent_health_check__"),c.push({name:"BigQuery API",category:"bigquery",status:"ok",message:"BigQuery API is reachable"});}catch(x){let R=x?.message??String(x),P=R.toLowerCase(),v=P.includes("disabled")||P.includes("has not been used")||P.includes("accessnotconfigured"),h=P.includes("permission")||R.includes("403")||P.includes("access denied"),_=P.includes("project")&&P.includes("not found"),ne=P.includes("not found")||R.includes("404");v?c.push({name:"BigQuery API",category:"bigquery",status:"error",message:"BigQuery API is not enabled",fix:{gcloud:`gcloud services enable bigquery.googleapis.com --project=${t}`,console:`${d}/apis/library/bigquery.googleapis.com?project=${t}`}}):_?c.push({name:"BigQuery Project",category:"bigquery",status:"error",message:R,fix:{hint:"The GCP project does not exist or the credentials don't have access to it. In the Firebase emulator, GCLOUD_PROJECT may override the configured projectId. Ensure you pass the correct projectId to the BigQuery constructor and have valid credentials.",console:`${d}/home/dashboard`}}):h?c.push({name:"BigQuery API",category:"bigquery",status:"error",message:`Permission denied: ${R}`,fix:{hint:"Grant the service account BigQuery Data Editor + BigQuery Job User roles",gcloud:[`SA=$(gcloud run services describe YOUR_SERVICE --region=YOUR_REGION --format="value(spec.template.spec.serviceAccountName)" --project=${t})`,`gcloud projects add-iam-policy-binding ${t} --member="serviceAccount:$SA" --role="roles/bigquery.dataEditor"`,`gcloud projects add-iam-policy-binding ${t} --member="serviceAccount:$SA" --role="roles/bigquery.jobUser"`].join(`
|
|
76
|
+
`),console:`${d}/iam-admin/iam?project=${t}`}}):ne?c.push({name:"BigQuery Dataset",category:"bigquery",status:"error",message:`Dataset not found: ${R}`,fix:{hint:"Create the dataset first",gcloud:`bq mk --dataset ${t}:YOUR_DATASET_ID`,console:`${d}/bigquery?project=${t}`}}):c.push({name:"BigQuery API",category:"bigquery",status:"ok",message:"BigQuery API is reachable (table lookup returned expected error)"});}for(let x of b)try{let R=await e.tableExists(x.tableName);c.push({name:`Table: ${x.tableName}`,category:"bigquery",status:R?"ok":"warn",message:R?`Table \`${x.tableName}\` exists`:`Table \`${x.tableName}\` does not exist yet`,...!R&&{fix:{hint:"Table will be auto-created on first sync if autoMigrate is enabled. Or create it manually."}}});}catch(R){c.push({name:`Table: ${x.tableName}`,category:"bigquery",status:"error",message:R?.message??String(R)});}if(m)for(let x of b){let R=`${p}-${x.name}`;try{let P=m.topic(R);if(typeof P.exists=="function"){let[v]=await P.exists();c.push({name:`Topic: ${R}`,category:"pubsub",status:v?"ok":"error",message:v?`Topic \`${R}\` exists`:`Topic \`${R}\` does not exist`,...!v&&{fix:{gcloud:`gcloud pubsub topics create ${R} --project=${t}`,console:`${d}/cloudpubsub/topic/list?project=${t}`}}});}else c.push({name:`Topic: ${R}`,category:"pubsub",status:"warn",message:"Cannot verify topic existence (PubSub client doesn't expose .exists())",fix:{gcloud:`gcloud pubsub topics create ${R} --project=${t}`,console:`${d}/cloudpubsub/topic/list?project=${t}`,hint:"Ensure the topic exists. It is auto-created by the Firebase emulator but must exist in production."}});}catch(P){let v=P?.message??String(P),h=v.includes("disabled")||v.includes("has not been used");if(c.push({name:h?"Pub/Sub API":`Topic: ${R}`,category:"pubsub",status:"error",message:h?"Pub/Sub API is not enabled":v,fix:h?{gcloud:`gcloud services enable pubsub.googleapis.com --project=${t}`,console:`${d}/apis/library/pubsub.googleapis.com?project=${t}`}:{gcloud:`gcloud pubsub topics create ${R} --project=${t}`,console:`${d}/cloudpubsub/topic/list?project=${t}`}}),h)break}}else c.push({name:"Pub/Sub Client",category:"pubsub",status:"warn",message:"PubSub client not available for config check"});if(j(l)){let x=c.every(R=>R.status==="ok");q(a,{project:t,healthy:x,checks:c});return}let f=x=>x==="ok"?'<span class="badge badge-ok">OK</span>':x==="warn"?'<span class="badge badge-warn">WARN</span>':'<span class="badge badge-err">ERROR</span>',C={bigquery:c.filter(x=>x.category==="bigquery"),pubsub:c.filter(x=>x.category==="pubsub"),firestore:c.filter(x=>x.category==="firestore")},w=(x,R)=>{if(R.length===0)return "";let P=R.map(v=>{let h="";if(v.fix){let _=[];v.fix.hint&&_.push(`<p class="muted">${v.fix.hint}</p>`),v.fix.gcloud&&_.push(`<pre>$ ${v.fix.gcloud}</pre>`),v.fix.console&&_.push(`<p><a href="${v.fix.console}" target="_blank">Open GCP Console \u2192</a></p>`),h=`<div style="margin-top:.5rem">${_.join("")}</div>`;}return `<tr>
|
|
77
|
+
<td>${f(v.status)}</td>
|
|
78
|
+
<td><strong>${v.name}</strong><br><span class="muted">${v.message}</span>${h}</td>
|
|
79
79
|
</tr>`}).join(`
|
|
80
|
-
`);return `<h2>${
|
|
80
|
+
`);return `<h2>${x}</h2>
|
|
81
81
|
<table><thead><tr><th style="width:80px">Status</th><th>Check</th></tr></thead>
|
|
82
|
-
<tbody>${
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
82
|
+
<tbody>${P}</tbody></table>`},T=c.every(x=>x.status==="ok")?'<span class="badge badge-ok">All checks passed</span>':'<span class="badge badge-warn">Some issues found</span>',D=m?`<form method="POST" action="${s}/config-check/setup-pubsub" style="display:inline">
|
|
83
|
+
<button type="submit" class="btn btn-primary">\u2699 Setup Pub/Sub (topics + subscriptions)</button>
|
|
84
|
+
</form>
|
|
85
|
+
<p class="muted" style="margin-top:.5rem">
|
|
86
|
+
Idempotent. Creates missing topics and subscriptions with
|
|
87
|
+
<code>enableMessageOrdering=${r.pubsubSetup?.ordering??true}</code>.
|
|
88
|
+
Existing subscriptions are kept as-is (the ordering flag is immutable).
|
|
89
|
+
</p>`:"",E=N("Config Check",s,`<div class="card">
|
|
90
|
+
<p>Project: <code>${t}</code> ${T}</p>
|
|
91
|
+
${w("BigQuery",C.bigquery)}
|
|
92
|
+
${w("Pub/Sub",C.pubsub)}
|
|
93
|
+
${w("Firestore",C.firestore)}
|
|
94
|
+
${D?`<hr style="margin:1.5rem 0"><h2>Actions</h2>${D}`:""}
|
|
95
|
+
</div>`);O(a,E);}),m&&S.post(`${u}/config-check/setup-pubsub`,async(l,a)=>{let s=I(l,u),t=r.pubsubSetup??{};try{let d=await Q(o,{pubsub:m,topicPrefix:$??"firestore-sync",ordering:t.ordering??!0,subscriptionSuffix:t.subscriptionSuffix??"sync-sub",includeDLQ:t.includeDLQ??!0,ackDeadlineSeconds:t.ackDeadlineSeconds??60,...t.messageRetentionDuration&&{messageRetentionDuration:t.messageRetentionDuration}});if(j(l)){q(a,{ok:!0,...d});return}let p=d.topics.map(f=>`<tr><td><code>${f.name}</code></td><td>${f.created?'<span class="badge badge-ok">created</span>':'<span class="badge">already exists</span>'}</td></tr>`).join(`
|
|
96
|
+
`),c=d.subscriptions.map(f=>`<tr>
|
|
97
|
+
<td><code>${f.name}</code></td>
|
|
98
|
+
<td><code>${f.topic}</code></td>
|
|
99
|
+
<td>${f.created?'<span class="badge badge-ok">created</span>':'<span class="badge">already exists</span>'}</td>
|
|
100
|
+
<td>${f.orderingEnabled?"\u2713":"\u2717"}</td>
|
|
101
|
+
<td>${f.warning?`<span class="badge badge-warn">${f.warning}</span>`:""}</td>
|
|
102
|
+
</tr>`).join(`
|
|
103
|
+
`);O(a,N("Pub/Sub Setup",s,`<div class="card">
|
|
104
|
+
<p><span class="badge badge-ok">Setup complete</span></p>
|
|
105
|
+
<h2>Topics</h2>
|
|
106
|
+
<table><thead><tr><th>Name</th><th>Status</th></tr></thead>
|
|
107
|
+
<tbody>${p}</tbody></table>
|
|
108
|
+
<h2>Subscriptions</h2>
|
|
109
|
+
<table><thead><tr><th>Name</th><th>Topic</th><th>Status</th><th>Ordering</th><th>Notes</th></tr></thead>
|
|
110
|
+
<tbody>${c}</tbody></table>
|
|
111
|
+
<p style="margin-top:1rem"><a class="btn" href="${s}/config-check">\u2190 Back to Config Check</a></p>
|
|
112
|
+
</div>`));}catch(d){let p=d?.message??String(d);if(j(l)){q(a,{ok:false,error:p},500);return}O(a,N("Pub/Sub Setup \u2014 Error",s,`<div class="card">
|
|
113
|
+
<p><span class="badge badge-err">Setup failed</span></p>
|
|
114
|
+
<pre>${p}</pre>
|
|
115
|
+
<p><a class="btn" href="${s}/config-check">\u2190 Back</a></p>
|
|
116
|
+
</div>`),500);}})),async(l,a)=>{await S.handle(l,a);}}var ce="firestore-sync";function de(o,e){let n=e.ref?.path??void 0;return n?`${n}/{docId}`:(console.warn(`[SyncTriggers] Cannot determine collection path for "${o}". Skipping.`),null)}function B(o,e){let{onDocumentCreated:n,onDocumentUpdated:i,onDocumentDeleted:r}=e.deps.firestoreTriggers,g=e.deps.pubsub,m=e?.topicPrefix??ce,$={},u=e?.ordering,y=!!u,b=typeof u=="function"?u:u===true?s=>s.docId:null,S=new Map;function l(s){let t=S.get(s);return t||(t=y?g.topic(s,{messageOrdering:true}):g.topic(s),S.set(s,t),t)}async function a(s,t){let d=l(s),p=b?b(t):void 0;try{await d.publishMessage(p!==void 0?{json:t,orderingKey:p}:{json:t});}catch(c){throw p!==void 0&&typeof d.resumePublishing=="function"&&d.resumePublishing(p),c}}for(let[s,t]of Object.entries(o)){let d=e?.repos?.[s],p;if(t._isGroup){if(!d?.triggerPath){console.warn(`[SyncTriggers] Skipping collection-group repo "${s}". Provide a triggerPath in the sync repos config for group collections.`);continue}p=d.triggerPath;}else p=d?.triggerPath??de(s,t);if(!p)continue;let c=t._systemKeys?.[0]??"docId",f=`${m}-${s}`;$[`${s}_onCreate`]=n(p,async C=>{let w=C.data;if(!w)return;let k=w.data();if(!k)return;let T=String(k[c]??w.id),D=z(k,{exclude:d?.exclude,columnMap:d?.columnMap}),E={operation:"INSERT",repoName:s,docId:T,data:D,timestamp:new Date().toISOString()};await a(f,E);}),$[`${s}_onUpdate`]=i(p,async C=>{let w=C.data?.after;if(!w)return;let k=w.data();if(!k)return;let T=String(k[c]??w.id),D=z(k,{exclude:d?.exclude,columnMap:d?.columnMap}),E={operation:"UPSERT",repoName:s,docId:T,data:D,timestamp:new Date().toISOString()};await a(f,E);}),$[`${s}_onDelete`]=r(p,async C=>{let w=C.data;if(!w)return;let k=w.data(),T=String(k?.[c]??w.id),D={operation:"DELETE",repoName:s,docId:T,data:null,timestamp:new Date().toISOString()};await a(f,D);});}return $}var Z=class{constructor(e){this.buffer=[];this.flushing=false;this.timer=null;this.adapter=e.adapter,this.tableName=e.tableName,this.primaryKey=e.primaryKey,this.batchSize=e.batchSize??100,this.onFlushError=e.onFlushError;let n=e.flushIntervalMs??5e3;n>0&&(this.timer=setInterval(()=>{this.flush();},n),typeof this.timer=="object"&&"unref"in this.timer&&this.timer.unref());}get size(){return this.buffer.length}enqueue(...e){this.buffer.push(...e),this.buffer.length>=this.batchSize&&this.flush();}async flush(){if(this.flushing||this.buffer.length===0)return;this.flushing=true;let e=this.buffer.splice(0,this.batchSize);try{let n=[],i=[];for(let r of e)r.operation==="DELETE"?i.push(r.docId):r.data&&n.push(r.data);n.length>0&&await this.adapter.upsertRows(this.tableName,n,this.primaryKey),i.length>0&&await this.adapter.deleteRows(this.tableName,this.primaryKey,i);}catch(n){this.onFlushError?await this.onFlushError(e,n).catch(i=>{console.error(`[SyncQueue] Flush error for ${this.tableName}:`,n),console.error("[SyncQueue] Error handler also failed:",i);}):(this.buffer.unshift(...e),console.error(`[SyncQueue] Flush failed for ${this.tableName}:`,n));}finally{this.flushing=false;}}async shutdown(){this.timer&&(clearInterval(this.timer),this.timer=null),await this.flush();}};var X=new Set;async function ue(o,e,n,i,r,g,m){if(X.has(o))return;let $=A(n,e.dialect,{primaryKey:r,exclude:g,columnMap:m});if(!await e.tableExists(i))await e.createTable({tableName:i,columns:$});else {let y=new Set(await e.getTableColumns(i)),b=$.filter(S=>!y.has(S.name));b.length>0&&await e.addColumns(i,b);}X.add(o);}function H(o,e){let{deps:n,adapter:i,batchSize:r=100,flushIntervalMs:g=5e3,autoMigrate:m=false,topicPrefix:$="firestore-sync",repos:u={}}=e,y=new Map;function b(a,s){let t=y.get(a);if(t)return t;let p=u[a]?.tableName??a,c=async(f,C)=>{console.error(`[SyncWorker] Flush failed for "${a}" (${f.length} events):`,C);try{let w=`${$}-${a}-dlq`,k=n.pubsub.topic(w),[T]=await k.exists();T||(await k.create(),console.info(`[SyncWorker] Created DLQ topic "${w}"`));for(let D of f)await k.publishMessage({json:D});}catch(w){console.error(`[SyncWorker] Dead-letter publish also failed for ${a}:`,w);}};return t=new Z({adapter:i,tableName:p,primaryKey:s,batchSize:r,flushIntervalMs:g,onFlushError:c}),y.set(a,t),t}async function S(a){let{repoName:s}=a,t=o[s];if(!t){console.warn(`[SyncWorker] Unknown repo "${s}", skipping event`);return}let d=t._systemKeys?.[0]??t.documentKey??"docId",p=u[s],c=p?.columnMap,f=c?.[d]??d;if(m){let w=t.schema??void 0;if(w){let k=p?.tableName??s;await ue(s,i,w,k,d,p?.exclude,c);}}b(s,f).enqueue(a);}function l(a){return n.pubsubHandler.onMessagePublished(a,async s=>{let t=s.data?.message?.json??s.data?.json;if(!t){console.warn("[SyncWorker] Received empty PubSub message");return}await S(t);let d=y.get(t.repoName);d&&await d.flush();})}return {handleMessage:S,createHandler:l,queues:y,async shutdown(){let a=[];for(let s of y.values())a.push(s.shutdown());await Promise.all(a);}}}var le="firestore-sync";function ee(o){if(typeof o!="function")return o;let e=o,n;return new Proxy({},{get(i,r){return n||(n=e()),n[r]},has(i,r){return n||(n=e()),r in n}})}function pe(o,e){let{deps:n,adapter:i,topicPrefix:r=le,batchSize:g,flushIntervalMs:m,autoMigrate:$,admin:u,repos:y}=e,b=ee(n.pubsub),S=ee(i),l=B(o,{deps:{firestoreTriggers:n.firestoreTriggers,pubsub:b},topicPrefix:r,repos:y}),a=H(o,{deps:{pubsubHandler:n.pubsubHandler,pubsub:b},adapter:S,batchSize:g,flushIntervalMs:m,autoMigrate:$,topicPrefix:r,repos:y}),s={};for(let p of Object.keys(o))s[`sync_${p}`]=a.createHandler(`${r}-${p}`);let t=null;u&&(t=U(o,S,a.queues,a.handleMessage,u,y??{},b,r),s.adminsync=u.onRequest?u.httpsOptions?u.onRequest(u.httpsOptions,t):u.onRequest(t):t);let d={functions:{...l,...s},adminHandler:t,handleMessage:a.handleMessage,queues:a.queues,shutdown:a.shutdown};for(let p of ["adminHandler","handleMessage","queues","shutdown"])Object.defineProperty(d,p,{enumerable:false});return d}function te(o,e){let n=e.columns.map(i=>{let r=i.isPrimaryKey?" NOT NULL":"";return ` ${o.quoteIdentifier(i.name)} ${i.sqlType}${r}`}).join(`,
|
|
88
117
|
`);return `CREATE TABLE IF NOT EXISTS ${o.quoteIdentifier(e.tableName)} (
|
|
89
118
|
${n}
|
|
90
|
-
);`}function
|
|
91
|
-
`)}function
|
|
119
|
+
);`}function fe(o,e,n){return n.map(i=>`ALTER TABLE ${o.quoteIdentifier(e)} ADD COLUMN ${o.quoteIdentifier(i.name)} ${i.sqlType};`).join(`
|
|
120
|
+
`)}function ge(o,e,n){let i=[];for(let[r,g]of Object.entries(o)){let m=g.schema??g._schema??void 0;if(!m)continue;let $=n?.repos?.[r],u=$?.tableName??r,y=g._systemKeys?.[0]??g.documentKey??"docId",b=A(m,e,{primaryKey:y,exclude:$?.exclude,columnMap:$?.columnMap}),S={tableName:u,columns:b};i.push(te(e,S));}return i.join(`
|
|
92
121
|
|
|
93
|
-
`)}async function
|
|
122
|
+
`)}async function me(o,e,n){let i={created:[],altered:[],upToDate:[],skipped:[]};for(let[r,g]of Object.entries(o)){let m=g.schema??void 0;if(!m){i.skipped.push(r);continue}let $=n?.repos?.[r],u=$?.tableName??r,y=g._systemKeys?.[0]??g.documentKey??"docId",b=A(m,e.dialect,{primaryKey:y,exclude:$?.exclude,columnMap:$?.columnMap}),S={tableName:u,columns:b};if(!await e.tableExists(u))await e.createTable(S),i.created.push(u);else {let a=new Set(await e.getTableColumns(u)),s=b.filter(t=>!a.has(t.name));s.length>0?(await e.addColumns(u,s),i.altered.push(u)):i.upToDate.push(u);}}return i}export{Z as SyncQueue,fe as addColumnsDDL,me as autoMigrate,pe as createFirestoreSync,B as createSyncTriggers,H as createSyncWorker,te as createTableDDL,U as createadminsyncServer,Q as ensureSyncInfra,ge as generateDDL,z as serializeDocument,K as serializeValue,A as zodSchemaToColumns,ie as zodTypeToLogical};//# sourceMappingURL=index.js.map
|
|
94
123
|
//# sourceMappingURL=index.js.map
|