@lpdjs/firestore-repo-service 2.4.1 → 2.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/sync/bigquery.cjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
'use strict';var S=(s=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(s,{get:(t,e)=>(typeof require<"u"?require:t)[e]}):s)(function(s){if(typeof require<"u")return require.apply(this,arguments);throw Error('Dynamic require of "'+s+'" is not supported')});var u="__sync_version";function m(s){let t=s.toUpperCase();switch(t){case "INTEGER":return "INT64";case "FLOAT":return "FLOAT64";case "BOOLEAN":return "BOOL";default:return t}}var
|
|
1
|
+
'use strict';var S=(s=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(s,{get:(t,e)=>(typeof require<"u"?require:t)[e]}):s)(function(s){if(typeof require<"u")return require.apply(this,arguments);throw Error('Dynamic require of "'+s+'" is not supported')});var u="__sync_version";function m(s){let t=s.toUpperCase();switch(t){case "INTEGER":return "INT64";case "FLOAT":return "FLOAT64";case "BOOLEAN":return "BOOL";default:return t}}var y=class{constructor(){this.name="bigquery";}mapType(t){switch(t){case "string":return "STRING";case "number":return "FLOAT64";case "bigint":return "INT64";case "boolean":return "BOOL";case "timestamp":return "TIMESTAMP";case "json":return "JSON";case "text":return "STRING"}}quoteIdentifier(t){return `\`${t}\``}},T=new y,l=null;function E(){return l||(l=S("@google-cloud/bigquery-storage"),l)}function h(s){let t;return typeof s=="bigint"?t=s:typeof s=="number"||typeof s=="string"?t=BigInt(s):t=0n,t<0n&&(t=0n),t.toString(16).padStart(16,"0")}function C(s){if(!s||typeof s!="object")return false;let t=s;return t.code===4||t.code===10||t.code===13||t.code===14}async function I(s,t=6,e=200){let n=0;for(;;)try{return await s()}catch(r){if(n++,!C(r)||n>t)throw r;let i=e*Math.pow(2,n),a=Math.random()*i;await new Promise(g=>setTimeout(g,a));}}var c=class c{constructor(t){this.writers=new Map;this.bigquery=t.bigquery,this.projectId=t.projectId,this.datasetId=t.datasetId,this.maxStaleness=t.maxStaleness===void 0?"INTERVAL 15 MINUTE":t.maxStaleness,this.writerClient=t.writerClient;}get dialect(){return T}async tableExists(t){let[e]=await this.dataset.table(t).exists();return e}async getTableColumns(t){let[e]=await this.dataset.table(t).getMetadata();return (e.schema?.fields??[]).map(r=>r.name)}async getTableColumnsWithTypes(t){let[e]=await this.dataset.table(t).getMetadata(),n=e.schema?.fields??[],r=new Map;for(let i of n)r.set(i.name,m(i.type));return r}async createTable(t){let e=o=>this.dialect.quoteIdentifier(o),n=t.columns.find(o=>o.isPrimaryKey)?.name;if(!n)throw new Error(`BigQueryAdapter requires a primary key on table \`${t.tableName}\` (Storage Write CDC mode needs PRIMARY KEY NOT ENFORCED).`);let r=t.columns.map(o=>{let d=o.isPrimaryKey?" NOT NULL":"";return ` ${e(o.name)} ${o.sqlType}${d}`}).join(`,
|
|
2
2
|
`),i=[];this.maxStaleness!==null&&i.push(`max_staleness = ${this.maxStaleness}`);let a=i.length>0?`
|
|
3
|
-
OPTIONS(${i.join(", ")})`:"",
|
|
3
|
+
OPTIONS(${i.join(", ")})`:"",g=`CREATE TABLE IF NOT EXISTS ${this.fqn(t.tableName)} (
|
|
4
4
|
${r},
|
|
5
5
|
PRIMARY KEY (${e(n)}) NOT ENFORCED
|
|
6
6
|
)
|
|
7
|
-
CLUSTER BY ${e(n)}${a};`;await this.bigquery.query({query:
|
|
7
|
+
CLUSTER BY ${e(n)}${a};`;await this.bigquery.query({query:g});}async addColumns(t,e){let n=r=>this.dialect.quoteIdentifier(r);for(let r of e){let i=`ALTER TABLE ${this.fqn(t)} ADD COLUMN ${n(r.name)} ${r.sqlType};`;await this.bigquery.query({query:i});}}async executeRaw(t){await this.bigquery.query({query:t});}onSchemaChange(t){let e=this.writers.get(t);if(e){try{e.writer.close();}catch{}this.writers.delete(t);}}async insertRows(t,e){if(e.length===0)return;let n=await this.getOrCreateWriter(t),r=e.map(i=>({...this.normalizeRow(i),_CHANGE_TYPE:"UPSERT",_CHANGE_SEQUENCE_NUMBER:h(i[u])}));await this.appendWithRetry(t,n,r);}async upsertRows(t,e,n){if(e.length===0)return;let r=await this.getOrCreateWriter(t,n),i=e.map(a=>({...this.normalizeRow(a),_CHANGE_TYPE:"UPSERT",_CHANGE_SEQUENCE_NUMBER:h(a[u])}));await this.appendWithRetry(t,r,i);}async deleteRows(t,e,n){if(n.length===0)return;let r=await this.getOrCreateWriter(t,e),i=n.map(a=>({[e]:a,_CHANGE_TYPE:"DELETE",_CHANGE_SEQUENCE_NUMBER:"ffffffffffffffff"}));await this.appendWithRetry(t,r,i);}get dataset(){return this.bigquery.dataset(this.datasetId)}fqn(t){return `\`${this.datasetId}.${t}\``}static toEpochMicros(t){let e=t.getTime();return (BigInt(e)*1000n).toString()}normalizeRow(t){let e={};for(let[n,r]of Object.entries(t))if(r!==void 0)if(r instanceof Date)e[n]=c.toEpochMicros(r);else if(typeof r=="string"&&c.ISO_TIMESTAMP_RE.test(r)){let i=new Date(r);Number.isNaN(i.getTime())?e[n]=r:e[n]=c.toEpochMicros(i);}else typeof r=="object"&&r!==null?e[n]=JSON.stringify(r):typeof r=="bigint"?e[n]=r.toString():e[n]=r;return e}async appendWithRetry(t,e,n){await I(async()=>{try{await e.appendRows(n).getResult();}catch(r){throw this.onSchemaChange(t),r}});}async getOrCreateWriter(t,e){let n=this.writers.get(t);if(n)if(e&&n.primaryKey!==e)this.onSchemaChange(t);else return n.writer;let r=E();this.writerClient||(this.writerClient=new r.managedwriter.WriterClient({projectId:this.projectId}));let[i]=await this.dataset.table(t).getMetadata(),a={fields:i.schema?.fields??[]};e||(e=i.tableConstraints?.primaryKey?.columns?.[0]??a.fields[0]?.name);let g=r.adapt.convertBigQuerySchemaToStorageTableSchema(a),o=r.adapt.convertStorageSchemaToProto2Descriptor(g,"Row",r.adapt.withChangeType(),r.adapt.withChangeSequenceNumber()),d=`projects/${this.projectId}/datasets/${this.datasetId}/tables/${t}`,w=await this.writerClient.createStreamConnection({streamId:r.managedwriter.DefaultStream,destinationTable:d}),p=new r.managedwriter.JSONWriter({connection:w,protoDescriptor:o});return this.writers.set(t,{writer:p,primaryKey:e??""}),p}async close(){for(let{writer:t}of this.writers.values())try{t.close();}catch{}if(this.writers.clear(),this.writerClient&&typeof this.writerClient.close=="function")try{this.writerClient.close();}catch{}}};c.ISO_TIMESTAMP_RE=/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/;var f=c;exports.BigQueryAdapter=f;exports.bigqueryDialect=T;//# sourceMappingURL=bigquery.cjs.map
|
|
8
8
|
//# sourceMappingURL=bigquery.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/sync/constants.ts","../../src/sync/adapters/bigquery-types.ts","../../src/sync/adapters/bigquery.ts"],"names":["SYNC_VERSION_COLUMN","normalizeBigQueryType","type","upper","BigQueryDialect","logical","id","bigqueryDialect","storageNsCache","loadStorageNs","formatChangeSequenceNumber","version","n","isRetryableStorageError","err","e","withRetry","fn","maxRetries","baseMs","attempt","cap","delay","res","_BigQueryAdapter","options","tableName","exists","metadata","f","fields","result","table","qi","pk","c","cols","notNull","opts","optionsClause","ddl","columns","stmt","sql","cached","rows","writer","cdc","row","primaryKey","ids","d","ms","out","k","v","ns","bqSchema","storageSchema","protoDescriptor","destinationTable","connection","BigQueryAdapter"],"mappings":"sQAaO,IAAMA,CAAAA,CAAsB,iBCG5B,SAASC,CAAAA,CAAsBC,EAAsB,CAC1D,IAAMC,EAAQD,CAAAA,CAAK,WAAA,GACnB,OAAQC,CAAAA,EACN,KAAK,SAAA,CACH,OAAO,OAAA,CACT,KAAK,OAAA,CACH,OAAO,SAAA,CACT,KAAK,UACH,OAAO,MAAA,CACT,QACE,OAAOA,CACX,CACF,CC8BA,IAAMC,CAAAA,CAAN,KAA4C,CAA5C,WAAA,EAAA,CACE,KAAS,IAAA,CAAO,WAAA,CAEhB,QAAQC,CAAAA,CAA8B,CACpC,OAAQA,CAAAA,EACN,KAAK,QAAA,CACH,OAAO,QAAA,CACT,KAAK,QAAA,CACH,OAAO,UACT,KAAK,QAAA,CACH,OAAO,OAAA,CACT,KAAK,SAAA,CACH,OAAO,MAAA,CACT,KAAK,YACH,OAAO,WAAA,CACT,KAAK,MAAA,CACH,OAAO,OACT,KAAK,MAAA,CACH,OAAO,QACX,CACF,CAEA,gBAAgBC,CAAAA,CAAoB,CAClC,OAAO,CAAA,EAAA,EAAKA,CAAE,IAChB,CACF,CAAA,CAGaC,CAAAA,CAA8B,IAAIH,CAAAA,CAyB3CI,CAAAA,CAAmC,KACvC,SAASC,CAAAA,EAA2B,CAClC,OAAID,CAAAA,GAKJA,EADY,CAAA,CAAQ,gCAAgC,CAAA,CAE7CA,CAAAA,CACT,CAaA,SAASE,EAA2BC,CAAAA,CAA0B,CAC5D,IAAIC,CAAAA,CACJ,OAAI,OAAOD,CAAAA,EAAY,QAAA,CAAUC,CAAAA,CAAID,CAAAA,CAC5B,OAAOA,CAAAA,EAAY,UACnB,OAAOA,CAAAA,EAAY,SADUC,CAAAA,CAAI,MAAA,CAAOD,CAAO,CAAA,CAEnDC,CAAAA,CAAI,EAAA,CACLA,CAAAA,CAAI,EAAA,GAAIA,CAAAA,CAAI,IACTA,CAAAA,CAAE,QAAA,CAAS,EAAE,CAAA,CAAE,QAAA,CAAS,GAAI,GAAG,CACxC,CAOA,SAASC,CAAAA,CAAwBC,CAAAA,CAAuB,CACtD,GAAI,CAACA,GAAO,OAAOA,CAAAA,EAAQ,SAAU,OAAO,MAAA,CAC5C,IAAMC,CAAAA,CAAID,CAAAA,CAEV,OAAOC,EAAE,IAAA,GAAS,CAAA,EAAKA,EAAE,IAAA,GAAS,EAAA,EAAMA,EAAE,IAAA,GAAS,EAAA,EAAMA,CAAAA,CAAE,IAAA,GAAS,EACtE,CAEA,eAAeC,CAAAA,CACbC,CAAAA,CACAC,EAAa,CAAA,CACbC,CAAAA,CAAS,IACG,CACZ,IAAIC,CAAAA,CAAU,CAAA,CAEd,OACE,GAAI,CACF,OAAO,MAAMH,GACf,CAAA,MAASH,EAAK,CAEZ,GADAM,CAAAA,EAAAA,CACI,CAACP,CAAAA,CAAwBC,CAAG,GAAKM,CAAAA,CAAUF,CAAAA,CAAY,MAAMJ,CAAAA,CACjE,IAAMO,EAAMF,CAAAA,CAAS,IAAA,CAAK,GAAA,CAAI,CAAA,CAAGC,CAAO,CAAA,CAClCE,EAAQ,IAAA,CAAK,MAAA,GAAWD,CAAAA,CAC9B,MAAM,IAAI,OAAA,CAASE,CAAAA,EAAQ,WAAWA,CAAAA,CAAKD,CAAK,CAAC,EACnD,CAEJ,CA+DO,IAAME,CAAAA,CAAN,MAAMA,CAAsC,CAYjD,WAAA,CAAYC,CAAAA,CAAiC,CAL7C,IAAA,CAAiB,QAAU,IAAI,GAAA,CAM7B,KAAK,QAAA,CAAWA,CAAAA,CAAQ,SACxB,IAAA,CAAK,SAAA,CAAYA,CAAAA,CAAQ,SAAA,CACzB,IAAA,CAAK,SAAA,CAAYA,EAAQ,SAAA,CACzB,IAAA,CAAK,aACHA,CAAAA,CAAQ,YAAA,GAAiB,OACrB,oBAAA,CACAA,CAAAA,CAAQ,YAAA,CACd,IAAA,CAAK,YAAA,CAAeA,CAAAA,CAAQ,aAC9B,CAEA,IAAI,SAAsB,CACxB,OAAOlB,CACT,CAIA,MAAM,WAAA,CAAYmB,CAAAA,CAAqC,CACrD,GAAM,CAACC,CAAM,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAMD,CAAS,CAAA,CAAE,MAAA,EAAO,CAC5D,OAAOC,CACT,CAEA,MAAM,eAAA,CAAgBD,EAAsC,CAC1D,GAAM,CAACE,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,KAAA,CAAMF,CAAS,CAAA,CAAE,WAAA,GAEvD,OAAA,CADwCE,CAAAA,CAAS,QAAQ,MAAA,EAAU,EAAC,EACtD,GAAA,CAAKC,CAAAA,EAAMA,CAAAA,CAAE,IAAI,CACjC,CAEA,MAAM,wBAAA,CACJH,CAAAA,CAC8B,CAC9B,GAAM,CAACE,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,QAAQ,KAAA,CAAMF,CAAS,EAAE,WAAA,EAAY,CAC7DI,EACJF,CAAAA,CAAS,MAAA,EAAQ,MAAA,EAAU,EAAC,CACxBG,CAAAA,CAAS,IAAI,GAAA,CACnB,IAAA,IAAWF,KAAKC,CAAAA,CACdC,CAAAA,CAAO,IAAIF,CAAAA,CAAE,IAAA,CAAM5B,CAAAA,CAAsB4B,CAAAA,CAAE,IAAI,CAAC,EAElD,OAAOE,CACT,CAEA,MAAM,WAAA,CAAYC,EAAmC,CACnD,IAAMC,CAAAA,CAAM3B,CAAAA,EAAe,IAAA,CAAK,OAAA,CAAQ,gBAAgBA,CAAE,CAAA,CACpD4B,EAAKF,CAAAA,CAAM,OAAA,CAAQ,KAAMG,CAAAA,EAAMA,CAAAA,CAAE,YAAY,CAAA,EAAG,IAAA,CACtD,GAAI,CAACD,CAAAA,CACH,MAAM,IAAI,KAAA,CACR,CAAA,kDAAA,EAAqDF,EAAM,SAAS,CAAA,2DAAA,CAEtE,CAAA,CAEF,IAAMI,CAAAA,CAAOJ,CAAAA,CAAM,QAChB,GAAA,CAAKG,CAAAA,EAAM,CACV,IAAME,CAAAA,CAAUF,EAAE,YAAA,CAAe,WAAA,CAAc,EAAA,CAC/C,OAAO,CAAA,EAAA,EAAKF,CAAAA,CAAGE,EAAE,IAAI,CAAC,IAAIA,CAAAA,CAAE,OAAO,GAAGE,CAAO,CAAA,CAC/C,CAAC,CAAA,CACA,IAAA,CAAK,CAAA;AAAA,CAAK,EAEPC,CAAAA,CAAiB,GACnB,IAAA,CAAK,YAAA,GAAiB,MACxBA,CAAAA,CAAK,IAAA,CAAK,CAAA,gBAAA,EAAmB,IAAA,CAAK,YAAY,CAAA,CAAE,CAAA,CAElD,IAAMC,CAAAA,CACJD,CAAAA,CAAK,OAAS,CAAA,CAAI;AAAA,QAAA,EAAaA,CAAAA,CAAK,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,CAAM,EAAA,CAEhDE,CAAAA,CACJ,CAAA,2BAAA,EAA8B,IAAA,CAAK,GAAA,CAAIR,CAAAA,CAAM,SAAS,CAAC,CAAA;AAAA,EAAOI,CAAI,CAAA;AAAA,eAAA,EAChDH,CAAAA,CAAGC,CAAE,CAAC,CAAA;AAAA;AAAA,WAAA,EACRD,CAAAA,CAAGC,CAAE,CAAC,CAAA,EACnBK,CAAa,CAAA,CAAA,CAAA,CAElB,MAAM,IAAA,CAAK,QAAA,CAAS,KAAA,CAAM,CAAE,KAAA,CAAOC,CAAI,CAAC,EAC1C,CAEA,MAAM,UAAA,CAAWd,CAAAA,CAAmBe,CAAAA,CAAqC,CACvE,IAAMR,EAAM3B,CAAAA,EAAe,IAAA,CAAK,OAAA,CAAQ,eAAA,CAAgBA,CAAE,CAAA,CAC1D,IAAA,IAAW6B,CAAAA,IAAKM,CAAAA,CAAS,CACvB,IAAMC,CAAAA,CAAO,CAAA,YAAA,EAAe,IAAA,CAAK,GAAA,CAAIhB,CAAS,CAAC,CAAA,YAAA,EAAeO,EAAGE,CAAAA,CAAE,IAAI,CAAC,CAAA,CAAA,EAAIA,CAAAA,CAAE,OAAO,CAAA,CAAA,CAAA,CACrF,MAAM,KAAK,QAAA,CAAS,KAAA,CAAM,CAAE,KAAA,CAAOO,CAAK,CAAC,EAC3C,CACF,CAEA,MAAM,UAAA,CAAWC,CAAAA,CAA4B,CAC3C,MAAM,IAAA,CAAK,QAAA,CAAS,KAAA,CAAM,CAAE,MAAOA,CAAI,CAAC,EAC1C,CAOA,cAAA,CAAejB,CAAAA,CAAyB,CACtC,IAAMkB,EAAS,IAAA,CAAK,OAAA,CAAQ,GAAA,CAAIlB,CAAS,CAAA,CACzC,GAAIkB,CAAAA,CAAQ,CACV,GAAI,CACFA,CAAAA,CAAO,MAAA,CAAO,KAAA,GAChB,CAAA,KAAQ,CAER,CACA,KAAK,OAAA,CAAQ,MAAA,CAAOlB,CAAS,EAC/B,CACF,CAIA,MAAM,UAAA,CACJA,CAAAA,CACAmB,EACe,CACf,GAAIA,CAAAA,CAAK,MAAA,GAAW,CAAA,CAAG,OAGvB,IAAMC,CAAAA,CAAS,MAAM,IAAA,CAAK,iBAAA,CAAkBpB,CAAS,CAAA,CAC/CqB,EAAMF,CAAAA,CAAK,GAAA,CAAKG,CAAAA,GAAS,CAC7B,GAAG,IAAA,CAAK,YAAA,CAAaA,CAAG,CAAA,CACxB,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyBtC,CAAAA,CACvBsC,EAAIhD,CAAmB,CACzB,CACF,CAAA,CAAE,EACF,MAAM,IAAA,CAAK,eAAA,CAAgB0B,CAAAA,CAAWoB,EAAQC,CAAG,EACnD,CAEA,MAAM,UAAA,CACJrB,CAAAA,CACAmB,CAAAA,CACAI,CAAAA,CACe,CACf,GAAIJ,CAAAA,CAAK,MAAA,GAAW,CAAA,CAAG,OACvB,IAAMC,CAAAA,CAAS,MAAM,KAAK,iBAAA,CAAkBpB,CAAAA,CAAWuB,CAAU,CAAA,CAC3DF,CAAAA,CAAMF,CAAAA,CAAK,GAAA,CAAKG,CAAAA,GAAS,CAC7B,GAAG,IAAA,CAAK,YAAA,CAAaA,CAAG,EACxB,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyBtC,CAAAA,CACvBsC,EAAIhD,CAAmB,CACzB,CACF,CAAA,CAAE,CAAA,CACF,MAAM,IAAA,CAAK,eAAA,CAAgB0B,EAAWoB,CAAAA,CAAQC,CAAG,EACnD,CAEA,MAAM,UAAA,CACJrB,CAAAA,CACAuB,CAAAA,CACAC,CAAAA,CACe,CACf,GAAIA,CAAAA,CAAI,MAAA,GAAW,CAAA,CAAG,OACtB,IAAMJ,CAAAA,CAAS,MAAM,KAAK,iBAAA,CAAkBpB,CAAAA,CAAWuB,CAAU,CAAA,CAM3DF,EAAMG,CAAAA,CAAI,GAAA,CAAK5C,CAAAA,GAAQ,CAC3B,CAAC2C,CAAU,EAAG3C,CAAAA,CACd,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyB,kBAC3B,CAAA,CAAE,EACF,MAAM,IAAA,CAAK,eAAA,CAAgBoB,CAAAA,CAAWoB,CAAAA,CAAQC,CAAG,EACnD,CAIA,IAAY,OAAA,EAAU,CACpB,OAAO,IAAA,CAAK,QAAA,CAAS,OAAA,CAAQ,IAAA,CAAK,SAAS,CAC7C,CAEQ,GAAA,CAAIrB,CAAAA,CAA2B,CACrC,OAAO,CAAA,EAAA,EAAK,IAAA,CAAK,SAAS,CAAA,CAAA,EAAIA,CAAS,CAAA,EAAA,CACzC,CAcA,OAAe,aAAA,CAAcyB,CAAAA,CAAiB,CAC5C,IAAMC,CAAAA,CAAKD,EAAE,OAAA,EAAQ,CACrB,OAAA,CAAQ,MAAA,CAAOC,CAAE,CAAA,CAAI,KAAA,EAAO,QAAA,EAC9B,CAGQ,YAAA,CAAaJ,CAAAA,CAAuD,CAC1E,IAAMK,CAAAA,CAA+B,EAAC,CACtC,IAAA,GAAW,CAACC,CAAAA,CAAGC,CAAC,CAAA,GAAK,MAAA,CAAO,QAAQP,CAAG,CAAA,CACrC,GAAIO,CAAAA,GAAM,OACV,GAAIA,CAAAA,YAAa,IAAA,CAEfF,CAAAA,CAAIC,CAAC,CAAA,CAAI9B,CAAAA,CAAgB,aAAA,CAAc+B,CAAC,CAAA,CAAA,KAAA,GAExC,OAAOA,CAAAA,EAAM,QAAA,EACb/B,CAAAA,CAAgB,gBAAA,CAAiB,IAAA,CAAK+B,CAAC,EACvC,CAGA,IAAMJ,CAAAA,CAAI,IAAI,IAAA,CAAKI,CAAC,CAAA,CACf,MAAA,CAAO,MAAMJ,CAAAA,CAAE,OAAA,EAAS,CAAA,CAG3BE,EAAIC,CAAC,CAAA,CAAIC,CAAAA,CAFTF,CAAAA,CAAIC,CAAC,CAAA,CAAI9B,CAAAA,CAAgB,aAAA,CAAc2B,CAAC,EAI5C,CAAA,KAAW,OAAOI,CAAAA,EAAM,UAAYA,CAAAA,GAAM,IAAA,CAExCF,CAAAA,CAAIC,CAAC,CAAA,CAAI,IAAA,CAAK,SAAA,CAAUC,CAAC,EAChB,OAAOA,CAAAA,EAAM,QAAA,CACtBF,CAAAA,CAAIC,CAAC,CAAA,CAAIC,CAAAA,CAAE,QAAA,GAEXF,CAAAA,CAAIC,CAAC,CAAA,CAAIC,CAAAA,CAGb,OAAOF,CACT,CAEA,MAAc,eAAA,CACZ3B,EACAoB,CAAAA,CACAD,CAAAA,CACe,CACf,MAAM7B,CAAAA,CAAU,SAAY,CAC1B,GAAI,CAEF,MADgB8B,CAAAA,CAAO,UAAA,CAAWD,CAAI,CAAA,CACxB,SAAA,GAChB,CAAA,MAAS/B,EAAK,CAGZ,MAAA,IAAA,CAAK,cAAA,CAAeY,CAAS,CAAA,CACvBZ,CACR,CACF,CAAC,EACH,CAEA,MAAc,iBAAA,CACZY,CAAAA,CACAuB,EACc,CACd,IAAML,CAAAA,CAAS,IAAA,CAAK,QAAQ,GAAA,CAAIlB,CAAS,CAAA,CACzC,GAAIkB,CAAAA,CACF,GAAIK,CAAAA,EAAcL,CAAAA,CAAO,aAAeK,CAAAA,CAEtC,IAAA,CAAK,cAAA,CAAevB,CAAS,OAE7B,OAAOkB,CAAAA,CAAO,MAAA,CAIlB,IAAMY,EAAK/C,CAAAA,EAAc,CACpB,IAAA,CAAK,YAAA,GACR,IAAA,CAAK,YAAA,CAAe,IAAI+C,CAAAA,CAAG,cAAc,YAAA,CAAa,CACpD,SAAA,CAAW,IAAA,CAAK,SAClB,CAAC,CAAA,CAAA,CAGH,GAAM,CAAC5B,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,KAAA,CAAMF,CAAS,CAAA,CAAE,WAAA,GACjD+B,CAAAA,CAAW,CAAE,MAAA,CAAQ7B,CAAAA,CAAS,MAAA,EAAQ,MAAA,EAAU,EAAG,EACpDqB,CAAAA,GAIHA,CAAAA,CADErB,CAAAA,CAAS,gBAAA,EAAkB,UAAA,EAAY,OAAA,GAAU,CAAC,CAAA,EACjB6B,EAAS,MAAA,CAAO,CAAC,CAAA,EAAG,IAAA,CAAA,CAGzD,IAAMC,CAAAA,CACJF,CAAAA,CAAG,KAAA,CAAM,yCAAA,CAA0CC,CAAQ,CAAA,CACvDE,CAAAA,CAAkBH,CAAAA,CAAG,KAAA,CAAM,sCAAA,CAC/BE,CAAAA,CACA,KAAA,CACAF,CAAAA,CAAG,MAAM,cAAA,EAAe,CACxBA,CAAAA,CAAG,KAAA,CAAM,0BACX,CAAA,CAEMI,CAAAA,CAAmB,CAAA,SAAA,EAAY,KAAK,SAAS,CAAA,UAAA,EAAa,IAAA,CAAK,SAAS,CAAA,QAAA,EAAWlC,CAAS,CAAA,CAAA,CAM5FmC,CAAAA,CAAa,MAAM,IAAA,CAAK,YAAA,CAAa,sBAAA,CAAuB,CAChE,SAAUL,CAAAA,CAAG,aAAA,CAAc,aAAA,CAC3B,gBAAA,CAAAI,CACF,CAAC,CAAA,CAEKd,CAAAA,CAAS,IAAIU,CAAAA,CAAG,aAAA,CAAc,UAAA,CAAW,CAC7C,WAAAK,CAAAA,CACA,eAAA,CAAAF,CACF,CAAC,CAAA,CAED,OAAA,IAAA,CAAK,OAAA,CAAQ,GAAA,CAAIjC,EAAW,CAAE,MAAA,CAAAoB,CAAAA,CAAQ,UAAA,CAAYG,CAAAA,EAAc,EAAG,CAAC,CAAA,CAC7DH,CACT,CAGA,MAAM,KAAA,EAAuB,CAC3B,OAAW,CAAE,MAAA,CAAAA,CAAO,CAAA,GAAK,KAAK,OAAA,CAAQ,MAAA,EAAO,CAC3C,GAAI,CACFA,CAAAA,CAAO,KAAA,GACT,MAAQ,CAER,CAGF,GADA,IAAA,CAAK,QAAQ,KAAA,EAAM,CACf,IAAA,CAAK,YAAA,EAAgB,OAAO,IAAA,CAAK,YAAA,CAAa,KAAA,EAAU,UAAA,CAC1D,GAAI,CACF,IAAA,CAAK,YAAA,CAAa,QACpB,CAAA,KAAQ,CAER,CAEJ,CACF,CAAA,CArUatB,CAAAA,CAsLa,gBAAA,CACtB,sCAAA,KAvLSsC,CAAAA,CAANtC","file":"bigquery.cjs","sourcesContent":["/**\n * Internal constants shared between the worker, queue, schema mapper and\n * SQL adapters.\n */\n\n/**\n * Name of the SQL column that stores the publish-time `version` of each\n * sync event. Used by the worker to discard out-of-order PubSub deliveries\n * (the MERGE only updates rows when the incoming version is strictly\n * greater than the stored one).\n *\n * Two underscores prefix avoids collisions with user-defined fields.\n */\nexport const SYNC_VERSION_COLUMN = \"__sync_version\";\n","/**\n * BigQuery type-name utilities used by the {@link BigQueryAdapter}.\n *\n * - {@link normalizeBigQueryType} canonicalises BigQuery type strings so that\n * `INTEGER`/`INT64`, `FLOAT`/`FLOAT64`, `BOOLEAN`/`BOOL`, etc. compare\n * equal during schema-drift detection.\n * - {@link isBigQueryTypeCompatible} returns whether a desired type can\n * safely keep an existing column of another type — only widenings that\n * BigQuery accepts via `ALTER COLUMN SET DATA TYPE` are allowed.\n */\n\n/**\n * Canonicalise a BigQuery type name returned by `getMetadata().schema`\n * (which may use legacy aliases like `INTEGER` or `FLOAT`) so that it can\n * be compared against the type produced by `BigQueryDialect.mapType()`.\n */\nexport function normalizeBigQueryType(type: string): string {\n const upper = type.toUpperCase();\n switch (upper) {\n case \"INTEGER\":\n return \"INT64\";\n case \"FLOAT\":\n return \"FLOAT64\";\n case \"BOOLEAN\":\n return \"BOOL\";\n default:\n return upper;\n }\n}\n\n/**\n * Whether `desired` is the same as, or a safe widening of, `existing`.\n * The widenings mirror what BigQuery allows via\n * `ALTER COLUMN x SET DATA TYPE …` — see\n * https://cloud.google.com/bigquery/docs/managing-table-schemas#change_column_types\n */\nexport function isBigQueryTypeCompatible(\n existing: string,\n desired: string,\n): boolean {\n const a = normalizeBigQueryType(existing);\n const b = normalizeBigQueryType(desired);\n if (a === b) return true;\n\n const widenings: Record<string, string[]> = {\n INT64: [\"NUMERIC\", \"BIGNUMERIC\", \"FLOAT64\"],\n NUMERIC: [\"BIGNUMERIC\", \"FLOAT64\"],\n DATE: [\"DATETIME\", \"TIMESTAMP\"],\n DATETIME: [\"TIMESTAMP\"],\n };\n return widenings[a]?.includes(b) ?? false;\n}\n","/**\n * BigQuery adapter — streams Firestore changes to BigQuery via the\n * **Storage Write API** in CDC (Change Data Capture) mode.\n *\n * Why CDC over the legacy MERGE approach:\n *\n * - **No DML concurrency limit.** MERGE/DELETE DML is bounded by ≈ 2\n * concurrent statements per table; busy collections triggered\n * `Could not serialize access … due to concurrent update` errors.\n * The Storage Write API has no such bound, so multiple Cloud Function\n * instances can flush in parallel without conflicts.\n * - **Cheaper at scale.** Storage Write is roughly ~50% cheaper than\n * legacy streaming inserts, and free for the first 2 TiB / month.\n * - **Same idempotency model.** Each row carries `_CHANGE_SEQUENCE_NUMBER`\n * built from the existing `__sync_version` column, so out-of-order\n * PubSub deliveries are merged correctly by BigQuery.\n *\n * Requirements:\n *\n * - Destination tables **must** declare a `PRIMARY KEY (...) NOT ENFORCED`\n * constraint and be clustered on that key. {@link BigQueryAdapter.createTable}\n * handles both for tables managed by this library.\n * - Tables should be created with `OPTIONS(max_staleness = INTERVAL …)`\n * so the CDC merge runs in the background instead of on every read\n * — see {@link BigQueryAdapterOptions.maxStaleness}.\n * - Service account needs `bigquery.tables.updateData` (e.g. via the\n * `roles/bigquery.dataEditor` role).\n *\n * @example\n * ```ts\n * import { BigQuery } from \"@google-cloud/bigquery\";\n * import { BigQueryAdapter } from \"@lpdjs/firestore-repo-service/sync/bigquery\";\n *\n * const adapter = new BigQueryAdapter({\n * projectId: \"my-project\",\n * datasetId: \"firestore_sync\",\n * bigquery: new BigQuery({ projectId: \"my-project\" }),\n * maxStaleness: \"INTERVAL 15 MINUTE\",\n * });\n * ```\n */\n\nimport type { BigQuery } from \"@google-cloud/bigquery\";\nimport { SYNC_VERSION_COLUMN } from \"../constants\";\nimport type {\n LogicalType,\n SqlAdapter,\n SqlColumn,\n SqlDialect,\n SqlTableDef,\n} from \"../types\";\nimport { normalizeBigQueryType } from \"./bigquery-types\";\n\n// ---------------------------------------------------------------------------\n// Dialect\n// ---------------------------------------------------------------------------\n\n/** BigQuery SQL dialect mapping. */\nclass BigQueryDialect implements SqlDialect {\n readonly name = \"bigquery\";\n\n mapType(logical: LogicalType): string {\n switch (logical) {\n case \"string\":\n return \"STRING\";\n case \"number\":\n return \"FLOAT64\";\n case \"bigint\":\n return \"INT64\";\n case \"boolean\":\n return \"BOOL\";\n case \"timestamp\":\n return \"TIMESTAMP\";\n case \"json\":\n return \"JSON\";\n case \"text\":\n return \"STRING\";\n }\n }\n\n quoteIdentifier(id: string): string {\n return `\\`${id}\\``;\n }\n}\n\n/** Shared BigQuery dialect singleton. */\nexport const bigqueryDialect: SqlDialect = new BigQueryDialect();\n\n// ---------------------------------------------------------------------------\n// Storage Write API loader (kept loose so `@google-cloud/bigquery-storage`\n// stays an OPTIONAL peer dep)\n// ---------------------------------------------------------------------------\n\ntype StorageNs = {\n managedwriter: {\n WriterClient: new (opts?: any) => any;\n JSONWriter: new (params: { connection: any; protoDescriptor: any }) => any;\n DefaultStream: any;\n };\n adapt: {\n convertBigQuerySchemaToStorageTableSchema(schema: any): any;\n convertStorageSchemaToProto2Descriptor(\n schema: any,\n scope: string,\n ...opts: any[]\n ): any;\n withChangeType(): any;\n withChangeSequenceNumber(): any;\n };\n};\n\nlet storageNsCache: StorageNs | null = null;\nfunction loadStorageNs(): StorageNs {\n if (storageNsCache) return storageNsCache;\n // Lazy require so the library does not pull `@google-cloud/bigquery-storage`\n // unless this adapter is actually instantiated.\n // eslint-disable-next-line @typescript-eslint/no-require-imports\n const mod = require(\"@google-cloud/bigquery-storage\");\n storageNsCache = mod as StorageNs;\n return storageNsCache;\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\n\n/**\n * Format `__sync_version` (Firestore commit timestamp in microseconds since\n * epoch) as the 16-character hex string required by `_CHANGE_SEQUENCE_NUMBER`.\n *\n * BigQuery compares sequence numbers lexicographically, so a fixed-width hex\n * representation is mandatory for correct ordering.\n */\nfunction formatChangeSequenceNumber(version: unknown): string {\n let n: bigint;\n if (typeof version === \"bigint\") n = version;\n else if (typeof version === \"number\") n = BigInt(version);\n else if (typeof version === \"string\") n = BigInt(version);\n else n = 0n;\n if (n < 0n) n = 0n;\n return n.toString(16).padStart(16, \"0\");\n}\n\n/**\n * Returns true when the Storage Write append failed with a transient gRPC\n * status (UNAVAILABLE, DEADLINE_EXCEEDED, INTERNAL, ABORTED). Caller can\n * safely retry these.\n */\nfunction isRetryableStorageError(err: unknown): boolean {\n if (!err || typeof err !== \"object\") return false;\n const e = err as { code?: number };\n // gRPC codes: 4=DEADLINE_EXCEEDED, 10=ABORTED, 13=INTERNAL, 14=UNAVAILABLE\n return e.code === 4 || e.code === 10 || e.code === 13 || e.code === 14;\n}\n\nasync function withRetry<T>(\n fn: () => Promise<T>,\n maxRetries = 6,\n baseMs = 200,\n): Promise<T> {\n let attempt = 0;\n // eslint-disable-next-line no-constant-condition\n while (true) {\n try {\n return await fn();\n } catch (err) {\n attempt++;\n if (!isRetryableStorageError(err) || attempt > maxRetries) throw err;\n const cap = baseMs * Math.pow(2, attempt);\n const delay = Math.random() * cap;\n await new Promise((res) => setTimeout(res, delay));\n }\n }\n}\n\n// ---------------------------------------------------------------------------\n// Adapter\n// ---------------------------------------------------------------------------\n\nexport interface BigQueryAdapterOptions {\n /** GCP project id that owns the dataset. */\n projectId: string;\n /** BigQuery dataset id. */\n datasetId: string;\n /**\n * BigQuery client used for DDL operations (createTable, addColumns,\n * getMetadata, executeRaw). Storage Write only handles inserts.\n *\n * Typed as `BigQuery` from `@google-cloud/bigquery` (peer dependency —\n * the type-only import is erased at compile time so the runtime stays\n * free of that module unless the consumer instantiates a client).\n */\n bigquery: BigQuery;\n /**\n * Optional pre-built Storage Write `WriterClient`. When omitted a new\n * one is created lazily on first use.\n */\n writerClient?: any;\n /**\n * Value passed to `OPTIONS(max_staleness = ...)` on `createTable`.\n *\n * **Why this matters**: in BigQuery CDC mode, Storage Write API rows\n * land in a delta buffer. They become visible to queries only after a\n * MERGE applies them to the base table. Two paths exist:\n *\n * - **Background MERGE** — runs periodically in the background, paid\n * for as part of CDC pricing, and never blocks your readers.\n * - **Read-time MERGE** — every `SELECT` query merges the delta on the\n * fly before returning. Always-fresh reads but every query pays for\n * the merge work.\n *\n * `max_staleness` is the tolerated lag between a write and what reads\n * see. Setting it tells BigQuery: \"background MERGE is fine if reads\n * are at most this stale\". When the threshold is exceeded, the next\n * read triggers a one-shot merge.\n *\n * **The default `INTERVAL 0` means \"always read-time merge\"** — every\n * query forces a full merge of pending CDC writes. That makes reads\n * dramatically slower and more expensive on busy tables, so this\n * library defaults to a 15 minute window in production.\n *\n * Recommended:\n * - `INTERVAL 15 MINUTE` in production (good cost/freshness trade-off).\n * - `INTERVAL 1 MINUTE` in dev for quick visibility.\n * - Set to `null` to omit the option entirely (only do this when you\n * know what you are doing — you'll likely hit slow & expensive reads).\n *\n * @default \"INTERVAL 15 MINUTE\"\n */\n maxStaleness?: string | null;\n}\n\n/**\n * BigQuery implementation of {@link SqlAdapter} using the Storage Write API\n * in CDC mode. See module-level docstring for the rationale.\n */\nexport class BigQueryAdapter implements SqlAdapter {\n private readonly bigquery: BigQuery;\n private readonly projectId: string;\n private readonly datasetId: string;\n private readonly maxStaleness: string | null;\n private writerClient: any;\n /** Cache of `{ writer, primaryKey }` per table name. */\n private readonly writers = new Map<\n string,\n { writer: any; primaryKey: string }\n >();\n\n constructor(options: BigQueryAdapterOptions) {\n this.bigquery = options.bigquery;\n this.projectId = options.projectId;\n this.datasetId = options.datasetId;\n this.maxStaleness =\n options.maxStaleness === undefined\n ? \"INTERVAL 15 MINUTE\"\n : options.maxStaleness;\n this.writerClient = options.writerClient;\n }\n\n get dialect(): SqlDialect {\n return bigqueryDialect;\n }\n\n // -------- DDL (delegated to @google-cloud/bigquery) ----------------------\n\n async tableExists(tableName: string): Promise<boolean> {\n const [exists] = await this.dataset.table(tableName).exists();\n return exists;\n }\n\n async getTableColumns(tableName: string): Promise<string[]> {\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const fields: Array<{ name: string }> = metadata.schema?.fields ?? [];\n return fields.map((f) => f.name);\n }\n\n async getTableColumnsWithTypes(\n tableName: string,\n ): Promise<Map<string, string>> {\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const fields: Array<{ name: string; type: string }> =\n metadata.schema?.fields ?? [];\n const result = new Map<string, string>();\n for (const f of fields) {\n result.set(f.name, normalizeBigQueryType(f.type));\n }\n return result;\n }\n\n async createTable(table: SqlTableDef): Promise<void> {\n const qi = (id: string) => this.dialect.quoteIdentifier(id);\n const pk = table.columns.find((c) => c.isPrimaryKey)?.name;\n if (!pk) {\n throw new Error(\n `BigQueryAdapter requires a primary key on table \\`${table.tableName}\\` ` +\n `(Storage Write CDC mode needs PRIMARY KEY NOT ENFORCED).`,\n );\n }\n const cols = table.columns\n .map((c) => {\n const notNull = c.isPrimaryKey ? \" NOT NULL\" : \"\";\n return ` ${qi(c.name)} ${c.sqlType}${notNull}`;\n })\n .join(\",\\n\");\n\n const opts: string[] = [];\n if (this.maxStaleness !== null) {\n opts.push(`max_staleness = ${this.maxStaleness}`);\n }\n const optionsClause =\n opts.length > 0 ? `\\nOPTIONS(${opts.join(\", \")})` : \"\";\n\n const ddl =\n `CREATE TABLE IF NOT EXISTS ${this.fqn(table.tableName)} (\\n${cols},\\n` +\n ` PRIMARY KEY (${qi(pk)}) NOT ENFORCED\\n)` +\n `\\nCLUSTER BY ${qi(pk)}` +\n `${optionsClause};`;\n\n await this.bigquery.query({ query: ddl });\n }\n\n async addColumns(tableName: string, columns: SqlColumn[]): Promise<void> {\n const qi = (id: string) => this.dialect.quoteIdentifier(id);\n for (const c of columns) {\n const stmt = `ALTER TABLE ${this.fqn(tableName)} ADD COLUMN ${qi(c.name)} ${c.sqlType};`;\n await this.bigquery.query({ query: stmt });\n }\n }\n\n async executeRaw(sql: string): Promise<void> {\n await this.bigquery.query({ query: sql });\n }\n\n /**\n * Invalidate the cached writer for a table. Called by the worker after\n * `addColumns` so the next append rebuilds the proto descriptor against\n * the new schema.\n */\n onSchemaChange(tableName: string): void {\n const cached = this.writers.get(tableName);\n if (cached) {\n try {\n cached.writer.close();\n } catch {\n // ignore — connection may already be torn down\n }\n this.writers.delete(tableName);\n }\n }\n\n // -------- Inserts (Storage Write API) -----------------------------------\n\n async insertRows(\n tableName: string,\n rows: Record<string, unknown>[],\n ): Promise<void> {\n if (rows.length === 0) return;\n // Plain inserts have no PK matching, but in CDC mode every row needs a\n // _CHANGE_TYPE. We treat them as UPSERT.\n const writer = await this.getOrCreateWriter(tableName);\n const cdc = rows.map((row) => ({\n ...this.normalizeRow(row),\n _CHANGE_TYPE: \"UPSERT\",\n _CHANGE_SEQUENCE_NUMBER: formatChangeSequenceNumber(\n row[SYNC_VERSION_COLUMN],\n ),\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n async upsertRows(\n tableName: string,\n rows: Record<string, unknown>[],\n primaryKey: string,\n ): Promise<void> {\n if (rows.length === 0) return;\n const writer = await this.getOrCreateWriter(tableName, primaryKey);\n const cdc = rows.map((row) => ({\n ...this.normalizeRow(row),\n _CHANGE_TYPE: \"UPSERT\",\n _CHANGE_SEQUENCE_NUMBER: formatChangeSequenceNumber(\n row[SYNC_VERSION_COLUMN],\n ),\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n async deleteRows(\n tableName: string,\n primaryKey: string,\n ids: string[],\n ): Promise<void> {\n if (ids.length === 0) return;\n const writer = await this.getOrCreateWriter(tableName, primaryKey);\n // Storage Write CDC requires a value for every NOT-NULL column (the PK)\n // and a `_CHANGE_TYPE`. Other columns may be omitted; missing values are\n // interpreted as NULL. We pass the maximum sequence number so the DELETE\n // wins over any concurrent UPSERT for the same key — tombstones from the\n // queue carry no `__sync_version` of their own.\n const cdc = ids.map((id) => ({\n [primaryKey]: id,\n _CHANGE_TYPE: \"DELETE\",\n _CHANGE_SEQUENCE_NUMBER: \"ffffffffffffffff\",\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n // -------- Internal helpers ----------------------------------------------\n\n private get dataset() {\n return this.bigquery.dataset(this.datasetId);\n }\n\n private fqn(tableName: string): string {\n return `\\`${this.datasetId}.${tableName}\\``;\n }\n\n /** ISO 8601 timestamp pattern (e.g. 2026-03-29T20:59:27.394Z) */\n private static readonly ISO_TIMESTAMP_RE =\n /^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}/;\n\n /**\n * Convert a Date or epoch-millis number into the epoch-microseconds string\n * required by the Storage Write API for TIMESTAMP columns.\n *\n * The JSONWriter encodes through protobuf and a TIMESTAMP field is an\n * `int64`. Passing an ISO string would make `Long.fromString` throw\n * `interior hyphen` on the `-` characters.\n */\n private static toEpochMicros(d: Date): string {\n const ms = d.getTime();\n return (BigInt(ms) * 1000n).toString();\n }\n\n /** Convert JS values into shapes accepted by JSONWriter. */\n private normalizeRow(row: Record<string, unknown>): Record<string, unknown> {\n const out: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(row)) {\n if (v === undefined) continue;\n if (v instanceof Date) {\n // TIMESTAMP column → int64 epoch micros (as string to preserve precision)\n out[k] = BigQueryAdapter.toEpochMicros(v);\n } else if (\n typeof v === \"string\" &&\n BigQueryAdapter.ISO_TIMESTAMP_RE.test(v)\n ) {\n // ISO timestamp produced upstream by `serializeDocument` (Firestore\n // Timestamp → ISO string) — convert to epoch micros for protobuf.\n const d = new Date(v);\n if (!Number.isNaN(d.getTime())) {\n out[k] = BigQueryAdapter.toEpochMicros(d);\n } else {\n out[k] = v;\n }\n } else if (typeof v === \"object\" && v !== null) {\n // JSON columns: serialise nested objects/arrays\n out[k] = JSON.stringify(v);\n } else if (typeof v === \"bigint\") {\n out[k] = v.toString();\n } else {\n out[k] = v;\n }\n }\n return out;\n }\n\n private async appendWithRetry(\n tableName: string,\n writer: any,\n rows: Record<string, unknown>[],\n ): Promise<void> {\n await withRetry(async () => {\n try {\n const pending = writer.appendRows(rows);\n await pending.getResult();\n } catch (err) {\n // On certain errors (schema drift, broken connection) drop the\n // cached writer so the next call rebuilds it.\n this.onSchemaChange(tableName);\n throw err;\n }\n });\n }\n\n private async getOrCreateWriter(\n tableName: string,\n primaryKey?: string,\n ): Promise<any> {\n const cached = this.writers.get(tableName);\n if (cached) {\n if (primaryKey && cached.primaryKey !== primaryKey) {\n // PK changed — invalidate and rebuild\n this.onSchemaChange(tableName);\n } else {\n return cached.writer;\n }\n }\n\n const ns = loadStorageNs();\n if (!this.writerClient) {\n this.writerClient = new ns.managedwriter.WriterClient({\n projectId: this.projectId,\n });\n }\n\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const bqSchema = { fields: metadata.schema?.fields ?? [] };\n if (!primaryKey) {\n // Try to recover PK from table constraints if available\n const tableConstraintsPK =\n metadata.tableConstraints?.primaryKey?.columns?.[0];\n primaryKey = tableConstraintsPK ?? bqSchema.fields[0]?.name;\n }\n\n const storageSchema =\n ns.adapt.convertBigQuerySchemaToStorageTableSchema(bqSchema);\n const protoDescriptor = ns.adapt.convertStorageSchemaToProto2Descriptor(\n storageSchema,\n \"Row\",\n ns.adapt.withChangeType(),\n ns.adapt.withChangeSequenceNumber(),\n );\n\n const destinationTable = `projects/${this.projectId}/datasets/${this.datasetId}/tables/${tableName}`;\n // Pass `DefaultStream` as `streamId` (not `streamType`): the `_default`\n // stream is implicit on every table, so the client must short-circuit to\n // `${destinationTable}/streams/_default` instead of calling\n // `createWriteStream({ type: DEFAULT })` which BigQuery rejects with\n // `Unable to create a stream with type TYPE_UNSPECIFIED`.\n const connection = await this.writerClient.createStreamConnection({\n streamId: ns.managedwriter.DefaultStream,\n destinationTable,\n });\n\n const writer = new ns.managedwriter.JSONWriter({\n connection,\n protoDescriptor,\n });\n\n this.writers.set(tableName, { writer, primaryKey: primaryKey ?? \"\" });\n return writer;\n }\n\n /** Close all open writer connections. Useful for graceful shutdown. */\n async close(): Promise<void> {\n for (const { writer } of this.writers.values()) {\n try {\n writer.close();\n } catch {\n /* ignore */\n }\n }\n this.writers.clear();\n if (this.writerClient && typeof this.writerClient.close === \"function\") {\n try {\n this.writerClient.close();\n } catch {\n /* ignore */\n }\n }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"sources":["../../src/sync/constants.ts","../../src/sync/adapters/bigquery-types.ts","../../src/sync/adapters/bigquery.ts"],"names":["SYNC_VERSION_COLUMN","normalizeBigQueryType","type","upper","BigQueryDialect","logical","id","bigqueryDialect","storageNsCache","loadStorageNs","formatChangeSequenceNumber","version","n","isRetryableStorageError","err","e","withRetry","fn","maxRetries","baseMs","attempt","cap","delay","res","_BigQueryAdapter","options","tableName","exists","metadata","f","fields","result","table","qi","pk","c","cols","notNull","opts","optionsClause","ddl","columns","stmt","sql","cached","rows","writer","cdc","row","primaryKey","ids","d","ms","out","k","v","ns","bqSchema","storageSchema","protoDescriptor","destinationTable","connection","BigQueryAdapter"],"mappings":"sQAaO,IAAMA,CAAAA,CAAsB,iBCG5B,SAASC,CAAAA,CAAsBC,EAAsB,CAC1D,IAAMC,EAAQD,CAAAA,CAAK,WAAA,GACnB,OAAQC,CAAAA,EACN,KAAK,SAAA,CACH,OAAO,OAAA,CACT,KAAK,OAAA,CACH,OAAO,SAAA,CACT,KAAK,UACH,OAAO,MAAA,CACT,QACE,OAAOA,CACX,CACF,CC8BA,IAAMC,CAAAA,CAAN,KAA4C,CAA5C,WAAA,EAAA,CACE,KAAS,IAAA,CAAO,WAAA,CAEhB,QAAQC,CAAAA,CAA8B,CACpC,OAAQA,CAAAA,EACN,KAAK,QAAA,CACH,OAAO,QAAA,CACT,KAAK,QAAA,CACH,OAAO,UACT,KAAK,QAAA,CACH,OAAO,OAAA,CACT,KAAK,SAAA,CACH,OAAO,MAAA,CACT,KAAK,YACH,OAAO,WAAA,CACT,KAAK,MAAA,CACH,OAAO,OACT,KAAK,MAAA,CACH,OAAO,QACX,CACF,CAEA,gBAAgBC,CAAAA,CAAoB,CAClC,OAAO,CAAA,EAAA,EAAKA,CAAE,IAChB,CACF,CAAA,CAGaC,CAAAA,CAA8B,IAAIH,CAAAA,CAyB3CI,CAAAA,CAAmC,KACvC,SAASC,CAAAA,EAA2B,CAClC,OAAID,CAAAA,GAKJA,EADY,CAAA,CAAQ,gCAAgC,CAAA,CAE7CA,CAAAA,CACT,CAaA,SAASE,EAA2BC,CAAAA,CAA0B,CAC5D,IAAIC,CAAAA,CACJ,OAAI,OAAOD,CAAAA,EAAY,QAAA,CAAUC,CAAAA,CAAID,CAAAA,CAC5B,OAAOA,CAAAA,EAAY,UACnB,OAAOA,CAAAA,EAAY,SADUC,CAAAA,CAAI,MAAA,CAAOD,CAAO,CAAA,CAEnDC,CAAAA,CAAI,EAAA,CACLA,CAAAA,CAAI,EAAA,GAAIA,CAAAA,CAAI,IACTA,CAAAA,CAAE,QAAA,CAAS,EAAE,CAAA,CAAE,QAAA,CAAS,GAAI,GAAG,CACxC,CAOA,SAASC,CAAAA,CAAwBC,CAAAA,CAAuB,CACtD,GAAI,CAACA,GAAO,OAAOA,CAAAA,EAAQ,SAAU,OAAO,MAAA,CAC5C,IAAMC,CAAAA,CAAID,CAAAA,CAEV,OAAOC,EAAE,IAAA,GAAS,CAAA,EAAKA,EAAE,IAAA,GAAS,EAAA,EAAMA,EAAE,IAAA,GAAS,EAAA,EAAMA,CAAAA,CAAE,IAAA,GAAS,EACtE,CAEA,eAAeC,CAAAA,CACbC,CAAAA,CACAC,EAAa,CAAA,CACbC,CAAAA,CAAS,IACG,CACZ,IAAIC,CAAAA,CAAU,CAAA,CAEd,OACE,GAAI,CACF,OAAO,MAAMH,GACf,CAAA,MAASH,EAAK,CAEZ,GADAM,CAAAA,EAAAA,CACI,CAACP,CAAAA,CAAwBC,CAAG,GAAKM,CAAAA,CAAUF,CAAAA,CAAY,MAAMJ,CAAAA,CACjE,IAAMO,EAAMF,CAAAA,CAAS,IAAA,CAAK,GAAA,CAAI,CAAA,CAAGC,CAAO,CAAA,CAClCE,EAAQ,IAAA,CAAK,MAAA,GAAWD,CAAAA,CAC9B,MAAM,IAAI,OAAA,CAASE,CAAAA,EAAQ,WAAWA,CAAAA,CAAKD,CAAK,CAAC,EACnD,CAEJ,CA0FO,IAAME,CAAAA,CAAN,MAAMA,CAAsC,CAYjD,WAAA,CAAYC,CAAAA,CAAiC,CAL7C,IAAA,CAAiB,QAAU,IAAI,GAAA,CAM7B,KAAK,QAAA,CAAWA,CAAAA,CAAQ,SACxB,IAAA,CAAK,SAAA,CAAYA,CAAAA,CAAQ,SAAA,CACzB,IAAA,CAAK,SAAA,CAAYA,EAAQ,SAAA,CACzB,IAAA,CAAK,aACHA,CAAAA,CAAQ,YAAA,GAAiB,OACrB,oBAAA,CACAA,CAAAA,CAAQ,YAAA,CACd,IAAA,CAAK,YAAA,CAAeA,CAAAA,CAAQ,aAC9B,CAEA,IAAI,SAAsB,CACxB,OAAOlB,CACT,CAIA,MAAM,WAAA,CAAYmB,CAAAA,CAAqC,CACrD,GAAM,CAACC,CAAM,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAMD,CAAS,CAAA,CAAE,MAAA,EAAO,CAC5D,OAAOC,CACT,CAEA,MAAM,eAAA,CAAgBD,EAAsC,CAC1D,GAAM,CAACE,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,KAAA,CAAMF,CAAS,CAAA,CAAE,WAAA,GAEvD,OAAA,CADwCE,CAAAA,CAAS,QAAQ,MAAA,EAAU,EAAC,EACtD,GAAA,CAAKC,CAAAA,EAAMA,CAAAA,CAAE,IAAI,CACjC,CAEA,MAAM,wBAAA,CACJH,CAAAA,CAC8B,CAC9B,GAAM,CAACE,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,QAAQ,KAAA,CAAMF,CAAS,EAAE,WAAA,EAAY,CAC7DI,EACJF,CAAAA,CAAS,MAAA,EAAQ,MAAA,EAAU,EAAC,CACxBG,CAAAA,CAAS,IAAI,GAAA,CACnB,IAAA,IAAWF,KAAKC,CAAAA,CACdC,CAAAA,CAAO,IAAIF,CAAAA,CAAE,IAAA,CAAM5B,CAAAA,CAAsB4B,CAAAA,CAAE,IAAI,CAAC,EAElD,OAAOE,CACT,CAEA,MAAM,WAAA,CAAYC,EAAmC,CACnD,IAAMC,CAAAA,CAAM3B,CAAAA,EAAe,IAAA,CAAK,OAAA,CAAQ,gBAAgBA,CAAE,CAAA,CACpD4B,EAAKF,CAAAA,CAAM,OAAA,CAAQ,KAAMG,CAAAA,EAAMA,CAAAA,CAAE,YAAY,CAAA,EAAG,IAAA,CACtD,GAAI,CAACD,CAAAA,CACH,MAAM,IAAI,KAAA,CACR,CAAA,kDAAA,EAAqDF,EAAM,SAAS,CAAA,2DAAA,CAEtE,CAAA,CAEF,IAAMI,CAAAA,CAAOJ,CAAAA,CAAM,QAChB,GAAA,CAAKG,CAAAA,EAAM,CACV,IAAME,CAAAA,CAAUF,EAAE,YAAA,CAAe,WAAA,CAAc,EAAA,CAC/C,OAAO,CAAA,EAAA,EAAKF,CAAAA,CAAGE,EAAE,IAAI,CAAC,IAAIA,CAAAA,CAAE,OAAO,GAAGE,CAAO,CAAA,CAC/C,CAAC,CAAA,CACA,IAAA,CAAK,CAAA;AAAA,CAAK,EAEPC,CAAAA,CAAiB,GACnB,IAAA,CAAK,YAAA,GAAiB,MACxBA,CAAAA,CAAK,IAAA,CAAK,CAAA,gBAAA,EAAmB,IAAA,CAAK,YAAY,CAAA,CAAE,CAAA,CAElD,IAAMC,CAAAA,CACJD,CAAAA,CAAK,OAAS,CAAA,CAAI;AAAA,QAAA,EAAaA,CAAAA,CAAK,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,CAAM,EAAA,CAEhDE,CAAAA,CACJ,CAAA,2BAAA,EAA8B,IAAA,CAAK,GAAA,CAAIR,CAAAA,CAAM,SAAS,CAAC,CAAA;AAAA,EAAOI,CAAI,CAAA;AAAA,eAAA,EAChDH,CAAAA,CAAGC,CAAE,CAAC,CAAA;AAAA;AAAA,WAAA,EACRD,CAAAA,CAAGC,CAAE,CAAC,CAAA,EACnBK,CAAa,CAAA,CAAA,CAAA,CAElB,MAAM,IAAA,CAAK,QAAA,CAAS,KAAA,CAAM,CAAE,KAAA,CAAOC,CAAI,CAAC,EAC1C,CAEA,MAAM,UAAA,CAAWd,CAAAA,CAAmBe,CAAAA,CAAqC,CACvE,IAAMR,EAAM3B,CAAAA,EAAe,IAAA,CAAK,OAAA,CAAQ,eAAA,CAAgBA,CAAE,CAAA,CAC1D,IAAA,IAAW6B,CAAAA,IAAKM,CAAAA,CAAS,CACvB,IAAMC,CAAAA,CAAO,CAAA,YAAA,EAAe,IAAA,CAAK,GAAA,CAAIhB,CAAS,CAAC,CAAA,YAAA,EAAeO,EAAGE,CAAAA,CAAE,IAAI,CAAC,CAAA,CAAA,EAAIA,CAAAA,CAAE,OAAO,CAAA,CAAA,CAAA,CACrF,MAAM,KAAK,QAAA,CAAS,KAAA,CAAM,CAAE,KAAA,CAAOO,CAAK,CAAC,EAC3C,CACF,CAEA,MAAM,UAAA,CAAWC,CAAAA,CAA4B,CAC3C,MAAM,IAAA,CAAK,QAAA,CAAS,KAAA,CAAM,CAAE,MAAOA,CAAI,CAAC,EAC1C,CAOA,cAAA,CAAejB,CAAAA,CAAyB,CACtC,IAAMkB,EAAS,IAAA,CAAK,OAAA,CAAQ,GAAA,CAAIlB,CAAS,CAAA,CACzC,GAAIkB,CAAAA,CAAQ,CACV,GAAI,CACFA,CAAAA,CAAO,MAAA,CAAO,KAAA,GAChB,CAAA,KAAQ,CAER,CACA,KAAK,OAAA,CAAQ,MAAA,CAAOlB,CAAS,EAC/B,CACF,CAIA,MAAM,UAAA,CACJA,CAAAA,CACAmB,EACe,CACf,GAAIA,CAAAA,CAAK,MAAA,GAAW,CAAA,CAAG,OAGvB,IAAMC,CAAAA,CAAS,MAAM,IAAA,CAAK,iBAAA,CAAkBpB,CAAS,CAAA,CAC/CqB,EAAMF,CAAAA,CAAK,GAAA,CAAKG,CAAAA,GAAS,CAC7B,GAAG,IAAA,CAAK,YAAA,CAAaA,CAAG,CAAA,CACxB,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyBtC,CAAAA,CACvBsC,EAAIhD,CAAmB,CACzB,CACF,CAAA,CAAE,EACF,MAAM,IAAA,CAAK,eAAA,CAAgB0B,CAAAA,CAAWoB,EAAQC,CAAG,EACnD,CAEA,MAAM,UAAA,CACJrB,CAAAA,CACAmB,CAAAA,CACAI,CAAAA,CACe,CACf,GAAIJ,CAAAA,CAAK,MAAA,GAAW,CAAA,CAAG,OACvB,IAAMC,CAAAA,CAAS,MAAM,KAAK,iBAAA,CAAkBpB,CAAAA,CAAWuB,CAAU,CAAA,CAC3DF,CAAAA,CAAMF,CAAAA,CAAK,GAAA,CAAKG,CAAAA,GAAS,CAC7B,GAAG,IAAA,CAAK,YAAA,CAAaA,CAAG,EACxB,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyBtC,CAAAA,CACvBsC,EAAIhD,CAAmB,CACzB,CACF,CAAA,CAAE,CAAA,CACF,MAAM,IAAA,CAAK,eAAA,CAAgB0B,EAAWoB,CAAAA,CAAQC,CAAG,EACnD,CAEA,MAAM,UAAA,CACJrB,CAAAA,CACAuB,CAAAA,CACAC,CAAAA,CACe,CACf,GAAIA,CAAAA,CAAI,MAAA,GAAW,CAAA,CAAG,OACtB,IAAMJ,CAAAA,CAAS,MAAM,KAAK,iBAAA,CAAkBpB,CAAAA,CAAWuB,CAAU,CAAA,CAM3DF,EAAMG,CAAAA,CAAI,GAAA,CAAK5C,CAAAA,GAAQ,CAC3B,CAAC2C,CAAU,EAAG3C,CAAAA,CACd,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyB,kBAC3B,CAAA,CAAE,EACF,MAAM,IAAA,CAAK,eAAA,CAAgBoB,CAAAA,CAAWoB,CAAAA,CAAQC,CAAG,EACnD,CAIA,IAAY,OAAA,EAAU,CACpB,OAAO,IAAA,CAAK,QAAA,CAAS,OAAA,CAAQ,IAAA,CAAK,SAAS,CAC7C,CAEQ,GAAA,CAAIrB,CAAAA,CAA2B,CACrC,OAAO,CAAA,EAAA,EAAK,IAAA,CAAK,SAAS,CAAA,CAAA,EAAIA,CAAS,CAAA,EAAA,CACzC,CAcA,OAAe,aAAA,CAAcyB,CAAAA,CAAiB,CAC5C,IAAMC,CAAAA,CAAKD,EAAE,OAAA,EAAQ,CACrB,OAAA,CAAQ,MAAA,CAAOC,CAAE,CAAA,CAAI,KAAA,EAAO,QAAA,EAC9B,CAGQ,YAAA,CAAaJ,CAAAA,CAAuD,CAC1E,IAAMK,CAAAA,CAA+B,EAAC,CACtC,IAAA,GAAW,CAACC,CAAAA,CAAGC,CAAC,CAAA,GAAK,MAAA,CAAO,QAAQP,CAAG,CAAA,CACrC,GAAIO,CAAAA,GAAM,OACV,GAAIA,CAAAA,YAAa,IAAA,CAEfF,CAAAA,CAAIC,CAAC,CAAA,CAAI9B,CAAAA,CAAgB,aAAA,CAAc+B,CAAC,CAAA,CAAA,KAAA,GAExC,OAAOA,CAAAA,EAAM,QAAA,EACb/B,CAAAA,CAAgB,gBAAA,CAAiB,IAAA,CAAK+B,CAAC,EACvC,CAGA,IAAMJ,CAAAA,CAAI,IAAI,IAAA,CAAKI,CAAC,CAAA,CACf,MAAA,CAAO,MAAMJ,CAAAA,CAAE,OAAA,EAAS,CAAA,CAG3BE,EAAIC,CAAC,CAAA,CAAIC,CAAAA,CAFTF,CAAAA,CAAIC,CAAC,CAAA,CAAI9B,CAAAA,CAAgB,aAAA,CAAc2B,CAAC,EAI5C,CAAA,KAAW,OAAOI,CAAAA,EAAM,UAAYA,CAAAA,GAAM,IAAA,CAExCF,CAAAA,CAAIC,CAAC,CAAA,CAAI,IAAA,CAAK,SAAA,CAAUC,CAAC,EAChB,OAAOA,CAAAA,EAAM,QAAA,CACtBF,CAAAA,CAAIC,CAAC,CAAA,CAAIC,CAAAA,CAAE,QAAA,GAEXF,CAAAA,CAAIC,CAAC,CAAA,CAAIC,CAAAA,CAGb,OAAOF,CACT,CAEA,MAAc,eAAA,CACZ3B,EACAoB,CAAAA,CACAD,CAAAA,CACe,CACf,MAAM7B,CAAAA,CAAU,SAAY,CAC1B,GAAI,CAEF,MADgB8B,CAAAA,CAAO,UAAA,CAAWD,CAAI,CAAA,CACxB,SAAA,GAChB,CAAA,MAAS/B,EAAK,CAGZ,MAAA,IAAA,CAAK,cAAA,CAAeY,CAAS,CAAA,CACvBZ,CACR,CACF,CAAC,EACH,CAEA,MAAc,iBAAA,CACZY,CAAAA,CACAuB,EACc,CACd,IAAML,CAAAA,CAAS,IAAA,CAAK,QAAQ,GAAA,CAAIlB,CAAS,CAAA,CACzC,GAAIkB,CAAAA,CACF,GAAIK,CAAAA,EAAcL,CAAAA,CAAO,aAAeK,CAAAA,CAEtC,IAAA,CAAK,cAAA,CAAevB,CAAS,OAE7B,OAAOkB,CAAAA,CAAO,MAAA,CAIlB,IAAMY,EAAK/C,CAAAA,EAAc,CACpB,IAAA,CAAK,YAAA,GACR,IAAA,CAAK,YAAA,CAAe,IAAI+C,CAAAA,CAAG,cAAc,YAAA,CAAa,CACpD,SAAA,CAAW,IAAA,CAAK,SAClB,CAAC,CAAA,CAAA,CAGH,GAAM,CAAC5B,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,KAAA,CAAMF,CAAS,CAAA,CAAE,WAAA,GACjD+B,CAAAA,CAAW,CAAE,MAAA,CAAQ7B,CAAAA,CAAS,MAAA,EAAQ,MAAA,EAAU,EAAG,EACpDqB,CAAAA,GAIHA,CAAAA,CADErB,CAAAA,CAAS,gBAAA,EAAkB,UAAA,EAAY,OAAA,GAAU,CAAC,CAAA,EACjB6B,EAAS,MAAA,CAAO,CAAC,CAAA,EAAG,IAAA,CAAA,CAGzD,IAAMC,CAAAA,CACJF,CAAAA,CAAG,KAAA,CAAM,yCAAA,CAA0CC,CAAQ,CAAA,CACvDE,CAAAA,CAAkBH,CAAAA,CAAG,KAAA,CAAM,sCAAA,CAC/BE,CAAAA,CACA,KAAA,CACAF,CAAAA,CAAG,MAAM,cAAA,EAAe,CACxBA,CAAAA,CAAG,KAAA,CAAM,0BACX,CAAA,CAEMI,CAAAA,CAAmB,CAAA,SAAA,EAAY,KAAK,SAAS,CAAA,UAAA,EAAa,IAAA,CAAK,SAAS,CAAA,QAAA,EAAWlC,CAAS,CAAA,CAAA,CAM5FmC,CAAAA,CAAa,MAAM,IAAA,CAAK,YAAA,CAAa,sBAAA,CAAuB,CAChE,SAAUL,CAAAA,CAAG,aAAA,CAAc,aAAA,CAC3B,gBAAA,CAAAI,CACF,CAAC,CAAA,CAEKd,CAAAA,CAAS,IAAIU,CAAAA,CAAG,aAAA,CAAc,UAAA,CAAW,CAC7C,WAAAK,CAAAA,CACA,eAAA,CAAAF,CACF,CAAC,CAAA,CAED,OAAA,IAAA,CAAK,OAAA,CAAQ,GAAA,CAAIjC,EAAW,CAAE,MAAA,CAAAoB,CAAAA,CAAQ,UAAA,CAAYG,CAAAA,EAAc,EAAG,CAAC,CAAA,CAC7DH,CACT,CAGA,MAAM,KAAA,EAAuB,CAC3B,OAAW,CAAE,MAAA,CAAAA,CAAO,CAAA,GAAK,KAAK,OAAA,CAAQ,MAAA,EAAO,CAC3C,GAAI,CACFA,CAAAA,CAAO,KAAA,GACT,MAAQ,CAER,CAGF,GADA,IAAA,CAAK,QAAQ,KAAA,EAAM,CACf,IAAA,CAAK,YAAA,EAAgB,OAAO,IAAA,CAAK,YAAA,CAAa,KAAA,EAAU,UAAA,CAC1D,GAAI,CACF,IAAA,CAAK,YAAA,CAAa,QACpB,CAAA,KAAQ,CAER,CAEJ,CACF,CAAA,CArUatB,CAAAA,CAsLa,gBAAA,CACtB,sCAAA,KAvLSsC,CAAAA,CAANtC","file":"bigquery.cjs","sourcesContent":["/**\n * Internal constants shared between the worker, queue, schema mapper and\n * SQL adapters.\n */\n\n/**\n * Name of the SQL column that stores the publish-time `version` of each\n * sync event. Used by the worker to discard out-of-order PubSub deliveries\n * (the MERGE only updates rows when the incoming version is strictly\n * greater than the stored one).\n *\n * Two underscores prefix avoids collisions with user-defined fields.\n */\nexport const SYNC_VERSION_COLUMN = \"__sync_version\";\n","/**\n * BigQuery type-name utilities used by the {@link BigQueryAdapter}.\n *\n * - {@link normalizeBigQueryType} canonicalises BigQuery type strings so that\n * `INTEGER`/`INT64`, `FLOAT`/`FLOAT64`, `BOOLEAN`/`BOOL`, etc. compare\n * equal during schema-drift detection.\n * - {@link isBigQueryTypeCompatible} returns whether a desired type can\n * safely keep an existing column of another type — only widenings that\n * BigQuery accepts via `ALTER COLUMN SET DATA TYPE` are allowed.\n */\n\n/**\n * Canonicalise a BigQuery type name returned by `getMetadata().schema`\n * (which may use legacy aliases like `INTEGER` or `FLOAT`) so that it can\n * be compared against the type produced by `BigQueryDialect.mapType()`.\n */\nexport function normalizeBigQueryType(type: string): string {\n const upper = type.toUpperCase();\n switch (upper) {\n case \"INTEGER\":\n return \"INT64\";\n case \"FLOAT\":\n return \"FLOAT64\";\n case \"BOOLEAN\":\n return \"BOOL\";\n default:\n return upper;\n }\n}\n\n/**\n * Whether `desired` is the same as, or a safe widening of, `existing`.\n * The widenings mirror what BigQuery allows via\n * `ALTER COLUMN x SET DATA TYPE …` — see\n * https://cloud.google.com/bigquery/docs/managing-table-schemas#change_column_types\n */\nexport function isBigQueryTypeCompatible(\n existing: string,\n desired: string,\n): boolean {\n const a = normalizeBigQueryType(existing);\n const b = normalizeBigQueryType(desired);\n if (a === b) return true;\n\n const widenings: Record<string, string[]> = {\n INT64: [\"NUMERIC\", \"BIGNUMERIC\", \"FLOAT64\"],\n NUMERIC: [\"BIGNUMERIC\", \"FLOAT64\"],\n DATE: [\"DATETIME\", \"TIMESTAMP\"],\n DATETIME: [\"TIMESTAMP\"],\n };\n return widenings[a]?.includes(b) ?? false;\n}\n","/**\n * BigQuery adapter — streams Firestore changes to BigQuery via the\n * **Storage Write API** in CDC (Change Data Capture) mode.\n *\n * Why CDC over the legacy MERGE approach:\n *\n * - **No DML concurrency limit.** MERGE/DELETE DML is bounded by ≈ 2\n * concurrent statements per table; busy collections triggered\n * `Could not serialize access … due to concurrent update` errors.\n * The Storage Write API has no such bound, so multiple Cloud Function\n * instances can flush in parallel without conflicts.\n * - **Cheaper at scale.** Storage Write is roughly ~50% cheaper than\n * legacy streaming inserts, and free for the first 2 TiB / month.\n * - **Same idempotency model.** Each row carries `_CHANGE_SEQUENCE_NUMBER`\n * built from the existing `__sync_version` column, so out-of-order\n * PubSub deliveries are merged correctly by BigQuery.\n *\n * Requirements:\n *\n * - Destination tables **must** declare a `PRIMARY KEY (...) NOT ENFORCED`\n * constraint and be clustered on that key. {@link BigQueryAdapter.createTable}\n * handles both for tables managed by this library.\n * - Tables should be created with `OPTIONS(max_staleness = INTERVAL …)`\n * so the CDC merge runs in the background instead of on every read\n * — see {@link BigQueryAdapterOptions.maxStaleness}.\n * - Service account needs `bigquery.tables.updateData` (e.g. via the\n * `roles/bigquery.dataEditor` role).\n *\n * @example\n * ```ts\n * import { BigQuery } from \"@google-cloud/bigquery\";\n * import { BigQueryAdapter } from \"@lpdjs/firestore-repo-service/sync/bigquery\";\n *\n * const adapter = new BigQueryAdapter({\n * projectId: \"my-project\",\n * datasetId: \"firestore_sync\",\n * bigquery: new BigQuery({ projectId: \"my-project\" }),\n * maxStaleness: \"INTERVAL 15 MINUTE\",\n * });\n * ```\n */\n\nimport type { BigQuery } from \"@google-cloud/bigquery\";\nimport { SYNC_VERSION_COLUMN } from \"../constants\";\nimport type {\n LogicalType,\n SqlAdapter,\n SqlColumn,\n SqlDialect,\n SqlTableDef,\n} from \"../types\";\nimport { normalizeBigQueryType } from \"./bigquery-types\";\n\n// ---------------------------------------------------------------------------\n// Dialect\n// ---------------------------------------------------------------------------\n\n/** BigQuery SQL dialect mapping. */\nclass BigQueryDialect implements SqlDialect {\n readonly name = \"bigquery\";\n\n mapType(logical: LogicalType): string {\n switch (logical) {\n case \"string\":\n return \"STRING\";\n case \"number\":\n return \"FLOAT64\";\n case \"bigint\":\n return \"INT64\";\n case \"boolean\":\n return \"BOOL\";\n case \"timestamp\":\n return \"TIMESTAMP\";\n case \"json\":\n return \"JSON\";\n case \"text\":\n return \"STRING\";\n }\n }\n\n quoteIdentifier(id: string): string {\n return `\\`${id}\\``;\n }\n}\n\n/** Shared BigQuery dialect singleton. */\nexport const bigqueryDialect: SqlDialect = new BigQueryDialect();\n\n// ---------------------------------------------------------------------------\n// Storage Write API loader (kept loose so `@google-cloud/bigquery-storage`\n// stays an OPTIONAL peer dep)\n// ---------------------------------------------------------------------------\n\ntype StorageNs = {\n managedwriter: {\n WriterClient: new (opts?: any) => any;\n JSONWriter: new (params: { connection: any; protoDescriptor: any }) => any;\n DefaultStream: any;\n };\n adapt: {\n convertBigQuerySchemaToStorageTableSchema(schema: any): any;\n convertStorageSchemaToProto2Descriptor(\n schema: any,\n scope: string,\n ...opts: any[]\n ): any;\n withChangeType(): any;\n withChangeSequenceNumber(): any;\n };\n};\n\nlet storageNsCache: StorageNs | null = null;\nfunction loadStorageNs(): StorageNs {\n if (storageNsCache) return storageNsCache;\n // Lazy require so the library does not pull `@google-cloud/bigquery-storage`\n // unless this adapter is actually instantiated.\n // eslint-disable-next-line @typescript-eslint/no-require-imports\n const mod = require(\"@google-cloud/bigquery-storage\");\n storageNsCache = mod as StorageNs;\n return storageNsCache;\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\n\n/**\n * Format `__sync_version` (Firestore commit timestamp in microseconds since\n * epoch) as the 16-character hex string required by `_CHANGE_SEQUENCE_NUMBER`.\n *\n * BigQuery compares sequence numbers lexicographically, so a fixed-width hex\n * representation is mandatory for correct ordering.\n */\nfunction formatChangeSequenceNumber(version: unknown): string {\n let n: bigint;\n if (typeof version === \"bigint\") n = version;\n else if (typeof version === \"number\") n = BigInt(version);\n else if (typeof version === \"string\") n = BigInt(version);\n else n = 0n;\n if (n < 0n) n = 0n;\n return n.toString(16).padStart(16, \"0\");\n}\n\n/**\n * Returns true when the Storage Write append failed with a transient gRPC\n * status (UNAVAILABLE, DEADLINE_EXCEEDED, INTERNAL, ABORTED). Caller can\n * safely retry these.\n */\nfunction isRetryableStorageError(err: unknown): boolean {\n if (!err || typeof err !== \"object\") return false;\n const e = err as { code?: number };\n // gRPC codes: 4=DEADLINE_EXCEEDED, 10=ABORTED, 13=INTERNAL, 14=UNAVAILABLE\n return e.code === 4 || e.code === 10 || e.code === 13 || e.code === 14;\n}\n\nasync function withRetry<T>(\n fn: () => Promise<T>,\n maxRetries = 6,\n baseMs = 200,\n): Promise<T> {\n let attempt = 0;\n // eslint-disable-next-line no-constant-condition\n while (true) {\n try {\n return await fn();\n } catch (err) {\n attempt++;\n if (!isRetryableStorageError(err) || attempt > maxRetries) throw err;\n const cap = baseMs * Math.pow(2, attempt);\n const delay = Math.random() * cap;\n await new Promise((res) => setTimeout(res, delay));\n }\n }\n}\n\n// ---------------------------------------------------------------------------\n// Adapter\n// ---------------------------------------------------------------------------\n\n/**\n * Minimal structural shape of `@google-cloud/bigquery`'s `BigQuery` client\n * actually used by the adapter (`.dataset(...)` and `.query(...)`).\n *\n * Why structural and not just `BigQuery`: TypeScript treats classes with\n * private fields nominally. When a consumer's `node_modules` ends up with\n * a *second* copy of `@google-cloud/bigquery` (very common in workspace\n * setups, peer-dep dedup misses, monorepo nesting), the `BigQuery` they\n * import is *technically* a different type than the one this library's\n * `BigQuery` references — even though they are runtime-identical — and\n * assignment fails with `Types have separate declarations of a private\n * property '_universeDomain'`.\n *\n * Accepting a structural superset sidesteps the duplicate-install issue\n * without giving up type safety on the methods we actually call.\n */\nexport interface BigQueryLike {\n dataset(datasetId: string): {\n table(tableName: string): any;\n [key: string]: any;\n };\n query(options: { query: string; params?: unknown[] } | string): Promise<any>;\n}\n\nexport interface BigQueryAdapterOptions {\n /** GCP project id that owns the dataset. */\n projectId: string;\n /** BigQuery dataset id. */\n datasetId: string;\n /**\n * BigQuery client used for DDL operations (createTable, addColumns,\n * getMetadata, executeRaw). Storage Write only handles inserts.\n *\n * Typed as the structural {@link BigQueryLike} (a superset satisfied by\n * `BigQuery` from `@google-cloud/bigquery`) to avoid TypeScript nominal\n * mismatches when the consumer's project ends up with two copies of\n * `@google-cloud/bigquery` in different `node_modules`. Pass\n * `new BigQuery({...})` from your own install — it satisfies this shape\n * structurally.\n */\n bigquery: BigQueryLike | BigQuery;\n /**\n * Optional pre-built Storage Write `WriterClient`. When omitted a new\n * one is created lazily on first use.\n */\n writerClient?: any;\n /**\n * Value passed to `OPTIONS(max_staleness = ...)` on `createTable`.\n *\n * **Why this matters**: in BigQuery CDC mode, Storage Write API rows\n * land in a delta buffer. They become visible to queries only after a\n * MERGE applies them to the base table. Two paths exist:\n *\n * - **Background MERGE** — runs periodically in the background, paid\n * for as part of CDC pricing, and never blocks your readers.\n * - **Read-time MERGE** — every `SELECT` query merges the delta on the\n * fly before returning. Always-fresh reads but every query pays for\n * the merge work.\n *\n * `max_staleness` is the tolerated lag between a write and what reads\n * see. Setting it tells BigQuery: \"background MERGE is fine if reads\n * are at most this stale\". When the threshold is exceeded, the next\n * read triggers a one-shot merge.\n *\n * **The default `INTERVAL 0` means \"always read-time merge\"** — every\n * query forces a full merge of pending CDC writes. That makes reads\n * dramatically slower and more expensive on busy tables, so this\n * library defaults to a 15 minute window in production.\n *\n * Recommended:\n * - `INTERVAL 15 MINUTE` in production (good cost/freshness trade-off).\n * - `INTERVAL 1 MINUTE` in dev for quick visibility.\n * - Set to `null` to omit the option entirely (only do this when you\n * know what you are doing — you'll likely hit slow & expensive reads).\n *\n * @default \"INTERVAL 15 MINUTE\"\n */\n maxStaleness?: string | null;\n}\n\n/**\n * BigQuery implementation of {@link SqlAdapter} using the Storage Write API\n * in CDC mode. See module-level docstring for the rationale.\n */\nexport class BigQueryAdapter implements SqlAdapter {\n private readonly bigquery: BigQueryLike;\n private readonly projectId: string;\n private readonly datasetId: string;\n private readonly maxStaleness: string | null;\n private writerClient: any;\n /** Cache of `{ writer, primaryKey }` per table name. */\n private readonly writers = new Map<\n string,\n { writer: any; primaryKey: string }\n >();\n\n constructor(options: BigQueryAdapterOptions) {\n this.bigquery = options.bigquery;\n this.projectId = options.projectId;\n this.datasetId = options.datasetId;\n this.maxStaleness =\n options.maxStaleness === undefined\n ? \"INTERVAL 15 MINUTE\"\n : options.maxStaleness;\n this.writerClient = options.writerClient;\n }\n\n get dialect(): SqlDialect {\n return bigqueryDialect;\n }\n\n // -------- DDL (delegated to @google-cloud/bigquery) ----------------------\n\n async tableExists(tableName: string): Promise<boolean> {\n const [exists] = await this.dataset.table(tableName).exists();\n return exists;\n }\n\n async getTableColumns(tableName: string): Promise<string[]> {\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const fields: Array<{ name: string }> = metadata.schema?.fields ?? [];\n return fields.map((f) => f.name);\n }\n\n async getTableColumnsWithTypes(\n tableName: string,\n ): Promise<Map<string, string>> {\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const fields: Array<{ name: string; type: string }> =\n metadata.schema?.fields ?? [];\n const result = new Map<string, string>();\n for (const f of fields) {\n result.set(f.name, normalizeBigQueryType(f.type));\n }\n return result;\n }\n\n async createTable(table: SqlTableDef): Promise<void> {\n const qi = (id: string) => this.dialect.quoteIdentifier(id);\n const pk = table.columns.find((c) => c.isPrimaryKey)?.name;\n if (!pk) {\n throw new Error(\n `BigQueryAdapter requires a primary key on table \\`${table.tableName}\\` ` +\n `(Storage Write CDC mode needs PRIMARY KEY NOT ENFORCED).`,\n );\n }\n const cols = table.columns\n .map((c) => {\n const notNull = c.isPrimaryKey ? \" NOT NULL\" : \"\";\n return ` ${qi(c.name)} ${c.sqlType}${notNull}`;\n })\n .join(\",\\n\");\n\n const opts: string[] = [];\n if (this.maxStaleness !== null) {\n opts.push(`max_staleness = ${this.maxStaleness}`);\n }\n const optionsClause =\n opts.length > 0 ? `\\nOPTIONS(${opts.join(\", \")})` : \"\";\n\n const ddl =\n `CREATE TABLE IF NOT EXISTS ${this.fqn(table.tableName)} (\\n${cols},\\n` +\n ` PRIMARY KEY (${qi(pk)}) NOT ENFORCED\\n)` +\n `\\nCLUSTER BY ${qi(pk)}` +\n `${optionsClause};`;\n\n await this.bigquery.query({ query: ddl });\n }\n\n async addColumns(tableName: string, columns: SqlColumn[]): Promise<void> {\n const qi = (id: string) => this.dialect.quoteIdentifier(id);\n for (const c of columns) {\n const stmt = `ALTER TABLE ${this.fqn(tableName)} ADD COLUMN ${qi(c.name)} ${c.sqlType};`;\n await this.bigquery.query({ query: stmt });\n }\n }\n\n async executeRaw(sql: string): Promise<void> {\n await this.bigquery.query({ query: sql });\n }\n\n /**\n * Invalidate the cached writer for a table. Called by the worker after\n * `addColumns` so the next append rebuilds the proto descriptor against\n * the new schema.\n */\n onSchemaChange(tableName: string): void {\n const cached = this.writers.get(tableName);\n if (cached) {\n try {\n cached.writer.close();\n } catch {\n // ignore — connection may already be torn down\n }\n this.writers.delete(tableName);\n }\n }\n\n // -------- Inserts (Storage Write API) -----------------------------------\n\n async insertRows(\n tableName: string,\n rows: Record<string, unknown>[],\n ): Promise<void> {\n if (rows.length === 0) return;\n // Plain inserts have no PK matching, but in CDC mode every row needs a\n // _CHANGE_TYPE. We treat them as UPSERT.\n const writer = await this.getOrCreateWriter(tableName);\n const cdc = rows.map((row) => ({\n ...this.normalizeRow(row),\n _CHANGE_TYPE: \"UPSERT\",\n _CHANGE_SEQUENCE_NUMBER: formatChangeSequenceNumber(\n row[SYNC_VERSION_COLUMN],\n ),\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n async upsertRows(\n tableName: string,\n rows: Record<string, unknown>[],\n primaryKey: string,\n ): Promise<void> {\n if (rows.length === 0) return;\n const writer = await this.getOrCreateWriter(tableName, primaryKey);\n const cdc = rows.map((row) => ({\n ...this.normalizeRow(row),\n _CHANGE_TYPE: \"UPSERT\",\n _CHANGE_SEQUENCE_NUMBER: formatChangeSequenceNumber(\n row[SYNC_VERSION_COLUMN],\n ),\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n async deleteRows(\n tableName: string,\n primaryKey: string,\n ids: string[],\n ): Promise<void> {\n if (ids.length === 0) return;\n const writer = await this.getOrCreateWriter(tableName, primaryKey);\n // Storage Write CDC requires a value for every NOT-NULL column (the PK)\n // and a `_CHANGE_TYPE`. Other columns may be omitted; missing values are\n // interpreted as NULL. We pass the maximum sequence number so the DELETE\n // wins over any concurrent UPSERT for the same key — tombstones from the\n // queue carry no `__sync_version` of their own.\n const cdc = ids.map((id) => ({\n [primaryKey]: id,\n _CHANGE_TYPE: \"DELETE\",\n _CHANGE_SEQUENCE_NUMBER: \"ffffffffffffffff\",\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n // -------- Internal helpers ----------------------------------------------\n\n private get dataset() {\n return this.bigquery.dataset(this.datasetId);\n }\n\n private fqn(tableName: string): string {\n return `\\`${this.datasetId}.${tableName}\\``;\n }\n\n /** ISO 8601 timestamp pattern (e.g. 2026-03-29T20:59:27.394Z) */\n private static readonly ISO_TIMESTAMP_RE =\n /^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}/;\n\n /**\n * Convert a Date or epoch-millis number into the epoch-microseconds string\n * required by the Storage Write API for TIMESTAMP columns.\n *\n * The JSONWriter encodes through protobuf and a TIMESTAMP field is an\n * `int64`. Passing an ISO string would make `Long.fromString` throw\n * `interior hyphen` on the `-` characters.\n */\n private static toEpochMicros(d: Date): string {\n const ms = d.getTime();\n return (BigInt(ms) * 1000n).toString();\n }\n\n /** Convert JS values into shapes accepted by JSONWriter. */\n private normalizeRow(row: Record<string, unknown>): Record<string, unknown> {\n const out: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(row)) {\n if (v === undefined) continue;\n if (v instanceof Date) {\n // TIMESTAMP column → int64 epoch micros (as string to preserve precision)\n out[k] = BigQueryAdapter.toEpochMicros(v);\n } else if (\n typeof v === \"string\" &&\n BigQueryAdapter.ISO_TIMESTAMP_RE.test(v)\n ) {\n // ISO timestamp produced upstream by `serializeDocument` (Firestore\n // Timestamp → ISO string) — convert to epoch micros for protobuf.\n const d = new Date(v);\n if (!Number.isNaN(d.getTime())) {\n out[k] = BigQueryAdapter.toEpochMicros(d);\n } else {\n out[k] = v;\n }\n } else if (typeof v === \"object\" && v !== null) {\n // JSON columns: serialise nested objects/arrays\n out[k] = JSON.stringify(v);\n } else if (typeof v === \"bigint\") {\n out[k] = v.toString();\n } else {\n out[k] = v;\n }\n }\n return out;\n }\n\n private async appendWithRetry(\n tableName: string,\n writer: any,\n rows: Record<string, unknown>[],\n ): Promise<void> {\n await withRetry(async () => {\n try {\n const pending = writer.appendRows(rows);\n await pending.getResult();\n } catch (err) {\n // On certain errors (schema drift, broken connection) drop the\n // cached writer so the next call rebuilds it.\n this.onSchemaChange(tableName);\n throw err;\n }\n });\n }\n\n private async getOrCreateWriter(\n tableName: string,\n primaryKey?: string,\n ): Promise<any> {\n const cached = this.writers.get(tableName);\n if (cached) {\n if (primaryKey && cached.primaryKey !== primaryKey) {\n // PK changed — invalidate and rebuild\n this.onSchemaChange(tableName);\n } else {\n return cached.writer;\n }\n }\n\n const ns = loadStorageNs();\n if (!this.writerClient) {\n this.writerClient = new ns.managedwriter.WriterClient({\n projectId: this.projectId,\n });\n }\n\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const bqSchema = { fields: metadata.schema?.fields ?? [] };\n if (!primaryKey) {\n // Try to recover PK from table constraints if available\n const tableConstraintsPK =\n metadata.tableConstraints?.primaryKey?.columns?.[0];\n primaryKey = tableConstraintsPK ?? bqSchema.fields[0]?.name;\n }\n\n const storageSchema =\n ns.adapt.convertBigQuerySchemaToStorageTableSchema(bqSchema);\n const protoDescriptor = ns.adapt.convertStorageSchemaToProto2Descriptor(\n storageSchema,\n \"Row\",\n ns.adapt.withChangeType(),\n ns.adapt.withChangeSequenceNumber(),\n );\n\n const destinationTable = `projects/${this.projectId}/datasets/${this.datasetId}/tables/${tableName}`;\n // Pass `DefaultStream` as `streamId` (not `streamType`): the `_default`\n // stream is implicit on every table, so the client must short-circuit to\n // `${destinationTable}/streams/_default` instead of calling\n // `createWriteStream({ type: DEFAULT })` which BigQuery rejects with\n // `Unable to create a stream with type TYPE_UNSPECIFIED`.\n const connection = await this.writerClient.createStreamConnection({\n streamId: ns.managedwriter.DefaultStream,\n destinationTable,\n });\n\n const writer = new ns.managedwriter.JSONWriter({\n connection,\n protoDescriptor,\n });\n\n this.writers.set(tableName, { writer, primaryKey: primaryKey ?? \"\" });\n return writer;\n }\n\n /** Close all open writer connections. Useful for graceful shutdown. */\n async close(): Promise<void> {\n for (const { writer } of this.writers.values()) {\n try {\n writer.close();\n } catch {\n /* ignore */\n }\n }\n this.writers.clear();\n if (this.writerClient && typeof this.writerClient.close === \"function\") {\n try {\n this.writerClient.close();\n } catch {\n /* ignore */\n }\n }\n }\n}\n"]}
|
package/dist/sync/bigquery.d.cts
CHANGED
|
@@ -49,6 +49,32 @@ import 'firebase-functions/v2/pubsub';
|
|
|
49
49
|
|
|
50
50
|
/** Shared BigQuery dialect singleton. */
|
|
51
51
|
declare const bigqueryDialect: SqlDialect;
|
|
52
|
+
/**
|
|
53
|
+
* Minimal structural shape of `@google-cloud/bigquery`'s `BigQuery` client
|
|
54
|
+
* actually used by the adapter (`.dataset(...)` and `.query(...)`).
|
|
55
|
+
*
|
|
56
|
+
* Why structural and not just `BigQuery`: TypeScript treats classes with
|
|
57
|
+
* private fields nominally. When a consumer's `node_modules` ends up with
|
|
58
|
+
* a *second* copy of `@google-cloud/bigquery` (very common in workspace
|
|
59
|
+
* setups, peer-dep dedup misses, monorepo nesting), the `BigQuery` they
|
|
60
|
+
* import is *technically* a different type than the one this library's
|
|
61
|
+
* `BigQuery` references — even though they are runtime-identical — and
|
|
62
|
+
* assignment fails with `Types have separate declarations of a private
|
|
63
|
+
* property '_universeDomain'`.
|
|
64
|
+
*
|
|
65
|
+
* Accepting a structural superset sidesteps the duplicate-install issue
|
|
66
|
+
* without giving up type safety on the methods we actually call.
|
|
67
|
+
*/
|
|
68
|
+
interface BigQueryLike {
|
|
69
|
+
dataset(datasetId: string): {
|
|
70
|
+
table(tableName: string): any;
|
|
71
|
+
[key: string]: any;
|
|
72
|
+
};
|
|
73
|
+
query(options: {
|
|
74
|
+
query: string;
|
|
75
|
+
params?: unknown[];
|
|
76
|
+
} | string): Promise<any>;
|
|
77
|
+
}
|
|
52
78
|
interface BigQueryAdapterOptions {
|
|
53
79
|
/** GCP project id that owns the dataset. */
|
|
54
80
|
projectId: string;
|
|
@@ -58,11 +84,14 @@ interface BigQueryAdapterOptions {
|
|
|
58
84
|
* BigQuery client used for DDL operations (createTable, addColumns,
|
|
59
85
|
* getMetadata, executeRaw). Storage Write only handles inserts.
|
|
60
86
|
*
|
|
61
|
-
* Typed as
|
|
62
|
-
*
|
|
63
|
-
*
|
|
87
|
+
* Typed as the structural {@link BigQueryLike} (a superset satisfied by
|
|
88
|
+
* `BigQuery` from `@google-cloud/bigquery`) to avoid TypeScript nominal
|
|
89
|
+
* mismatches when the consumer's project ends up with two copies of
|
|
90
|
+
* `@google-cloud/bigquery` in different `node_modules`. Pass
|
|
91
|
+
* `new BigQuery({...})` from your own install — it satisfies this shape
|
|
92
|
+
* structurally.
|
|
64
93
|
*/
|
|
65
|
-
bigquery: BigQuery;
|
|
94
|
+
bigquery: BigQueryLike | BigQuery;
|
|
66
95
|
/**
|
|
67
96
|
* Optional pre-built Storage Write `WriterClient`. When omitted a new
|
|
68
97
|
* one is created lazily on first use.
|
|
@@ -151,4 +180,4 @@ declare class BigQueryAdapter implements SqlAdapter {
|
|
|
151
180
|
close(): Promise<void>;
|
|
152
181
|
}
|
|
153
182
|
|
|
154
|
-
export { BigQueryAdapter, type BigQueryAdapterOptions, bigqueryDialect };
|
|
183
|
+
export { BigQueryAdapter, type BigQueryAdapterOptions, type BigQueryLike, bigqueryDialect };
|
package/dist/sync/bigquery.d.ts
CHANGED
|
@@ -49,6 +49,32 @@ import 'firebase-functions/v2/pubsub';
|
|
|
49
49
|
|
|
50
50
|
/** Shared BigQuery dialect singleton. */
|
|
51
51
|
declare const bigqueryDialect: SqlDialect;
|
|
52
|
+
/**
|
|
53
|
+
* Minimal structural shape of `@google-cloud/bigquery`'s `BigQuery` client
|
|
54
|
+
* actually used by the adapter (`.dataset(...)` and `.query(...)`).
|
|
55
|
+
*
|
|
56
|
+
* Why structural and not just `BigQuery`: TypeScript treats classes with
|
|
57
|
+
* private fields nominally. When a consumer's `node_modules` ends up with
|
|
58
|
+
* a *second* copy of `@google-cloud/bigquery` (very common in workspace
|
|
59
|
+
* setups, peer-dep dedup misses, monorepo nesting), the `BigQuery` they
|
|
60
|
+
* import is *technically* a different type than the one this library's
|
|
61
|
+
* `BigQuery` references — even though they are runtime-identical — and
|
|
62
|
+
* assignment fails with `Types have separate declarations of a private
|
|
63
|
+
* property '_universeDomain'`.
|
|
64
|
+
*
|
|
65
|
+
* Accepting a structural superset sidesteps the duplicate-install issue
|
|
66
|
+
* without giving up type safety on the methods we actually call.
|
|
67
|
+
*/
|
|
68
|
+
interface BigQueryLike {
|
|
69
|
+
dataset(datasetId: string): {
|
|
70
|
+
table(tableName: string): any;
|
|
71
|
+
[key: string]: any;
|
|
72
|
+
};
|
|
73
|
+
query(options: {
|
|
74
|
+
query: string;
|
|
75
|
+
params?: unknown[];
|
|
76
|
+
} | string): Promise<any>;
|
|
77
|
+
}
|
|
52
78
|
interface BigQueryAdapterOptions {
|
|
53
79
|
/** GCP project id that owns the dataset. */
|
|
54
80
|
projectId: string;
|
|
@@ -58,11 +84,14 @@ interface BigQueryAdapterOptions {
|
|
|
58
84
|
* BigQuery client used for DDL operations (createTable, addColumns,
|
|
59
85
|
* getMetadata, executeRaw). Storage Write only handles inserts.
|
|
60
86
|
*
|
|
61
|
-
* Typed as
|
|
62
|
-
*
|
|
63
|
-
*
|
|
87
|
+
* Typed as the structural {@link BigQueryLike} (a superset satisfied by
|
|
88
|
+
* `BigQuery` from `@google-cloud/bigquery`) to avoid TypeScript nominal
|
|
89
|
+
* mismatches when the consumer's project ends up with two copies of
|
|
90
|
+
* `@google-cloud/bigquery` in different `node_modules`. Pass
|
|
91
|
+
* `new BigQuery({...})` from your own install — it satisfies this shape
|
|
92
|
+
* structurally.
|
|
64
93
|
*/
|
|
65
|
-
bigquery: BigQuery;
|
|
94
|
+
bigquery: BigQueryLike | BigQuery;
|
|
66
95
|
/**
|
|
67
96
|
* Optional pre-built Storage Write `WriterClient`. When omitted a new
|
|
68
97
|
* one is created lazily on first use.
|
|
@@ -151,4 +180,4 @@ declare class BigQueryAdapter implements SqlAdapter {
|
|
|
151
180
|
close(): Promise<void>;
|
|
152
181
|
}
|
|
153
182
|
|
|
154
|
-
export { BigQueryAdapter, type BigQueryAdapterOptions, bigqueryDialect };
|
|
183
|
+
export { BigQueryAdapter, type BigQueryAdapterOptions, type BigQueryLike, bigqueryDialect };
|
package/dist/sync/bigquery.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
var S=(s=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(s,{get:(t,e)=>(typeof require<"u"?require:t)[e]}):s)(function(s){if(typeof require<"u")return require.apply(this,arguments);throw Error('Dynamic require of "'+s+'" is not supported')});var u="__sync_version";function m(s){let t=s.toUpperCase();switch(t){case "INTEGER":return "INT64";case "FLOAT":return "FLOAT64";case "BOOLEAN":return "BOOL";default:return t}}var
|
|
1
|
+
var S=(s=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(s,{get:(t,e)=>(typeof require<"u"?require:t)[e]}):s)(function(s){if(typeof require<"u")return require.apply(this,arguments);throw Error('Dynamic require of "'+s+'" is not supported')});var u="__sync_version";function m(s){let t=s.toUpperCase();switch(t){case "INTEGER":return "INT64";case "FLOAT":return "FLOAT64";case "BOOLEAN":return "BOOL";default:return t}}var y=class{constructor(){this.name="bigquery";}mapType(t){switch(t){case "string":return "STRING";case "number":return "FLOAT64";case "bigint":return "INT64";case "boolean":return "BOOL";case "timestamp":return "TIMESTAMP";case "json":return "JSON";case "text":return "STRING"}}quoteIdentifier(t){return `\`${t}\``}},T=new y,l=null;function E(){return l||(l=S("@google-cloud/bigquery-storage"),l)}function h(s){let t;return typeof s=="bigint"?t=s:typeof s=="number"||typeof s=="string"?t=BigInt(s):t=0n,t<0n&&(t=0n),t.toString(16).padStart(16,"0")}function C(s){if(!s||typeof s!="object")return false;let t=s;return t.code===4||t.code===10||t.code===13||t.code===14}async function I(s,t=6,e=200){let n=0;for(;;)try{return await s()}catch(r){if(n++,!C(r)||n>t)throw r;let i=e*Math.pow(2,n),a=Math.random()*i;await new Promise(g=>setTimeout(g,a));}}var c=class c{constructor(t){this.writers=new Map;this.bigquery=t.bigquery,this.projectId=t.projectId,this.datasetId=t.datasetId,this.maxStaleness=t.maxStaleness===void 0?"INTERVAL 15 MINUTE":t.maxStaleness,this.writerClient=t.writerClient;}get dialect(){return T}async tableExists(t){let[e]=await this.dataset.table(t).exists();return e}async getTableColumns(t){let[e]=await this.dataset.table(t).getMetadata();return (e.schema?.fields??[]).map(r=>r.name)}async getTableColumnsWithTypes(t){let[e]=await this.dataset.table(t).getMetadata(),n=e.schema?.fields??[],r=new Map;for(let i of n)r.set(i.name,m(i.type));return r}async createTable(t){let e=o=>this.dialect.quoteIdentifier(o),n=t.columns.find(o=>o.isPrimaryKey)?.name;if(!n)throw new Error(`BigQueryAdapter requires a primary key on table \`${t.tableName}\` (Storage Write CDC mode needs PRIMARY KEY NOT ENFORCED).`);let r=t.columns.map(o=>{let d=o.isPrimaryKey?" NOT NULL":"";return ` ${e(o.name)} ${o.sqlType}${d}`}).join(`,
|
|
2
2
|
`),i=[];this.maxStaleness!==null&&i.push(`max_staleness = ${this.maxStaleness}`);let a=i.length>0?`
|
|
3
|
-
OPTIONS(${i.join(", ")})`:"",
|
|
3
|
+
OPTIONS(${i.join(", ")})`:"",g=`CREATE TABLE IF NOT EXISTS ${this.fqn(t.tableName)} (
|
|
4
4
|
${r},
|
|
5
5
|
PRIMARY KEY (${e(n)}) NOT ENFORCED
|
|
6
6
|
)
|
|
7
|
-
CLUSTER BY ${e(n)}${a};`;await this.bigquery.query({query:
|
|
7
|
+
CLUSTER BY ${e(n)}${a};`;await this.bigquery.query({query:g});}async addColumns(t,e){let n=r=>this.dialect.quoteIdentifier(r);for(let r of e){let i=`ALTER TABLE ${this.fqn(t)} ADD COLUMN ${n(r.name)} ${r.sqlType};`;await this.bigquery.query({query:i});}}async executeRaw(t){await this.bigquery.query({query:t});}onSchemaChange(t){let e=this.writers.get(t);if(e){try{e.writer.close();}catch{}this.writers.delete(t);}}async insertRows(t,e){if(e.length===0)return;let n=await this.getOrCreateWriter(t),r=e.map(i=>({...this.normalizeRow(i),_CHANGE_TYPE:"UPSERT",_CHANGE_SEQUENCE_NUMBER:h(i[u])}));await this.appendWithRetry(t,n,r);}async upsertRows(t,e,n){if(e.length===0)return;let r=await this.getOrCreateWriter(t,n),i=e.map(a=>({...this.normalizeRow(a),_CHANGE_TYPE:"UPSERT",_CHANGE_SEQUENCE_NUMBER:h(a[u])}));await this.appendWithRetry(t,r,i);}async deleteRows(t,e,n){if(n.length===0)return;let r=await this.getOrCreateWriter(t,e),i=n.map(a=>({[e]:a,_CHANGE_TYPE:"DELETE",_CHANGE_SEQUENCE_NUMBER:"ffffffffffffffff"}));await this.appendWithRetry(t,r,i);}get dataset(){return this.bigquery.dataset(this.datasetId)}fqn(t){return `\`${this.datasetId}.${t}\``}static toEpochMicros(t){let e=t.getTime();return (BigInt(e)*1000n).toString()}normalizeRow(t){let e={};for(let[n,r]of Object.entries(t))if(r!==void 0)if(r instanceof Date)e[n]=c.toEpochMicros(r);else if(typeof r=="string"&&c.ISO_TIMESTAMP_RE.test(r)){let i=new Date(r);Number.isNaN(i.getTime())?e[n]=r:e[n]=c.toEpochMicros(i);}else typeof r=="object"&&r!==null?e[n]=JSON.stringify(r):typeof r=="bigint"?e[n]=r.toString():e[n]=r;return e}async appendWithRetry(t,e,n){await I(async()=>{try{await e.appendRows(n).getResult();}catch(r){throw this.onSchemaChange(t),r}});}async getOrCreateWriter(t,e){let n=this.writers.get(t);if(n)if(e&&n.primaryKey!==e)this.onSchemaChange(t);else return n.writer;let r=E();this.writerClient||(this.writerClient=new r.managedwriter.WriterClient({projectId:this.projectId}));let[i]=await this.dataset.table(t).getMetadata(),a={fields:i.schema?.fields??[]};e||(e=i.tableConstraints?.primaryKey?.columns?.[0]??a.fields[0]?.name);let g=r.adapt.convertBigQuerySchemaToStorageTableSchema(a),o=r.adapt.convertStorageSchemaToProto2Descriptor(g,"Row",r.adapt.withChangeType(),r.adapt.withChangeSequenceNumber()),d=`projects/${this.projectId}/datasets/${this.datasetId}/tables/${t}`,w=await this.writerClient.createStreamConnection({streamId:r.managedwriter.DefaultStream,destinationTable:d}),p=new r.managedwriter.JSONWriter({connection:w,protoDescriptor:o});return this.writers.set(t,{writer:p,primaryKey:e??""}),p}async close(){for(let{writer:t}of this.writers.values())try{t.close();}catch{}if(this.writers.clear(),this.writerClient&&typeof this.writerClient.close=="function")try{this.writerClient.close();}catch{}}};c.ISO_TIMESTAMP_RE=/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/;var f=c;export{f as BigQueryAdapter,T as bigqueryDialect};//# sourceMappingURL=bigquery.js.map
|
|
8
8
|
//# sourceMappingURL=bigquery.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/sync/constants.ts","../../src/sync/adapters/bigquery-types.ts","../../src/sync/adapters/bigquery.ts"],"names":["SYNC_VERSION_COLUMN","normalizeBigQueryType","type","upper","BigQueryDialect","logical","id","bigqueryDialect","storageNsCache","loadStorageNs","formatChangeSequenceNumber","version","n","isRetryableStorageError","err","e","withRetry","fn","maxRetries","baseMs","attempt","cap","delay","res","_BigQueryAdapter","options","tableName","exists","metadata","f","fields","result","table","qi","pk","c","cols","notNull","opts","optionsClause","ddl","columns","stmt","sql","cached","rows","writer","cdc","row","primaryKey","ids","d","ms","out","k","v","ns","bqSchema","storageSchema","protoDescriptor","destinationTable","connection","BigQueryAdapter"],"mappings":"yPAaO,IAAMA,CAAAA,CAAsB,iBCG5B,SAASC,CAAAA,CAAsBC,EAAsB,CAC1D,IAAMC,EAAQD,CAAAA,CAAK,WAAA,GACnB,OAAQC,CAAAA,EACN,KAAK,SAAA,CACH,OAAO,OAAA,CACT,KAAK,OAAA,CACH,OAAO,SAAA,CACT,KAAK,UACH,OAAO,MAAA,CACT,QACE,OAAOA,CACX,CACF,CC8BA,IAAMC,CAAAA,CAAN,KAA4C,CAA5C,WAAA,EAAA,CACE,KAAS,IAAA,CAAO,WAAA,CAEhB,QAAQC,CAAAA,CAA8B,CACpC,OAAQA,CAAAA,EACN,KAAK,QAAA,CACH,OAAO,QAAA,CACT,KAAK,QAAA,CACH,OAAO,UACT,KAAK,QAAA,CACH,OAAO,OAAA,CACT,KAAK,SAAA,CACH,OAAO,MAAA,CACT,KAAK,YACH,OAAO,WAAA,CACT,KAAK,MAAA,CACH,OAAO,OACT,KAAK,MAAA,CACH,OAAO,QACX,CACF,CAEA,gBAAgBC,CAAAA,CAAoB,CAClC,OAAO,CAAA,EAAA,EAAKA,CAAE,IAChB,CACF,CAAA,CAGaC,CAAAA,CAA8B,IAAIH,CAAAA,CAyB3CI,CAAAA,CAAmC,KACvC,SAASC,CAAAA,EAA2B,CAClC,OAAID,CAAAA,GAKJA,EADY,CAAA,CAAQ,gCAAgC,CAAA,CAE7CA,CAAAA,CACT,CAaA,SAASE,EAA2BC,CAAAA,CAA0B,CAC5D,IAAIC,CAAAA,CACJ,OAAI,OAAOD,CAAAA,EAAY,QAAA,CAAUC,CAAAA,CAAID,CAAAA,CAC5B,OAAOA,CAAAA,EAAY,UACnB,OAAOA,CAAAA,EAAY,SADUC,CAAAA,CAAI,MAAA,CAAOD,CAAO,CAAA,CAEnDC,CAAAA,CAAI,EAAA,CACLA,CAAAA,CAAI,EAAA,GAAIA,CAAAA,CAAI,IACTA,CAAAA,CAAE,QAAA,CAAS,EAAE,CAAA,CAAE,QAAA,CAAS,GAAI,GAAG,CACxC,CAOA,SAASC,CAAAA,CAAwBC,CAAAA,CAAuB,CACtD,GAAI,CAACA,GAAO,OAAOA,CAAAA,EAAQ,SAAU,OAAO,MAAA,CAC5C,IAAMC,CAAAA,CAAID,CAAAA,CAEV,OAAOC,EAAE,IAAA,GAAS,CAAA,EAAKA,EAAE,IAAA,GAAS,EAAA,EAAMA,EAAE,IAAA,GAAS,EAAA,EAAMA,CAAAA,CAAE,IAAA,GAAS,EACtE,CAEA,eAAeC,CAAAA,CACbC,CAAAA,CACAC,EAAa,CAAA,CACbC,CAAAA,CAAS,IACG,CACZ,IAAIC,CAAAA,CAAU,CAAA,CAEd,OACE,GAAI,CACF,OAAO,MAAMH,GACf,CAAA,MAASH,EAAK,CAEZ,GADAM,CAAAA,EAAAA,CACI,CAACP,CAAAA,CAAwBC,CAAG,GAAKM,CAAAA,CAAUF,CAAAA,CAAY,MAAMJ,CAAAA,CACjE,IAAMO,EAAMF,CAAAA,CAAS,IAAA,CAAK,GAAA,CAAI,CAAA,CAAGC,CAAO,CAAA,CAClCE,EAAQ,IAAA,CAAK,MAAA,GAAWD,CAAAA,CAC9B,MAAM,IAAI,OAAA,CAASE,CAAAA,EAAQ,WAAWA,CAAAA,CAAKD,CAAK,CAAC,EACnD,CAEJ,CA+DO,IAAME,CAAAA,CAAN,MAAMA,CAAsC,CAYjD,WAAA,CAAYC,CAAAA,CAAiC,CAL7C,IAAA,CAAiB,QAAU,IAAI,GAAA,CAM7B,KAAK,QAAA,CAAWA,CAAAA,CAAQ,SACxB,IAAA,CAAK,SAAA,CAAYA,CAAAA,CAAQ,SAAA,CACzB,IAAA,CAAK,SAAA,CAAYA,EAAQ,SAAA,CACzB,IAAA,CAAK,aACHA,CAAAA,CAAQ,YAAA,GAAiB,OACrB,oBAAA,CACAA,CAAAA,CAAQ,YAAA,CACd,IAAA,CAAK,YAAA,CAAeA,CAAAA,CAAQ,aAC9B,CAEA,IAAI,SAAsB,CACxB,OAAOlB,CACT,CAIA,MAAM,WAAA,CAAYmB,CAAAA,CAAqC,CACrD,GAAM,CAACC,CAAM,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAMD,CAAS,CAAA,CAAE,MAAA,EAAO,CAC5D,OAAOC,CACT,CAEA,MAAM,eAAA,CAAgBD,EAAsC,CAC1D,GAAM,CAACE,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,KAAA,CAAMF,CAAS,CAAA,CAAE,WAAA,GAEvD,OAAA,CADwCE,CAAAA,CAAS,QAAQ,MAAA,EAAU,EAAC,EACtD,GAAA,CAAKC,CAAAA,EAAMA,CAAAA,CAAE,IAAI,CACjC,CAEA,MAAM,wBAAA,CACJH,CAAAA,CAC8B,CAC9B,GAAM,CAACE,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,QAAQ,KAAA,CAAMF,CAAS,EAAE,WAAA,EAAY,CAC7DI,EACJF,CAAAA,CAAS,MAAA,EAAQ,MAAA,EAAU,EAAC,CACxBG,CAAAA,CAAS,IAAI,GAAA,CACnB,IAAA,IAAWF,KAAKC,CAAAA,CACdC,CAAAA,CAAO,IAAIF,CAAAA,CAAE,IAAA,CAAM5B,CAAAA,CAAsB4B,CAAAA,CAAE,IAAI,CAAC,EAElD,OAAOE,CACT,CAEA,MAAM,WAAA,CAAYC,EAAmC,CACnD,IAAMC,CAAAA,CAAM3B,CAAAA,EAAe,IAAA,CAAK,OAAA,CAAQ,gBAAgBA,CAAE,CAAA,CACpD4B,EAAKF,CAAAA,CAAM,OAAA,CAAQ,KAAMG,CAAAA,EAAMA,CAAAA,CAAE,YAAY,CAAA,EAAG,IAAA,CACtD,GAAI,CAACD,CAAAA,CACH,MAAM,IAAI,KAAA,CACR,CAAA,kDAAA,EAAqDF,EAAM,SAAS,CAAA,2DAAA,CAEtE,CAAA,CAEF,IAAMI,CAAAA,CAAOJ,CAAAA,CAAM,QAChB,GAAA,CAAKG,CAAAA,EAAM,CACV,IAAME,CAAAA,CAAUF,EAAE,YAAA,CAAe,WAAA,CAAc,EAAA,CAC/C,OAAO,CAAA,EAAA,EAAKF,CAAAA,CAAGE,EAAE,IAAI,CAAC,IAAIA,CAAAA,CAAE,OAAO,GAAGE,CAAO,CAAA,CAC/C,CAAC,CAAA,CACA,IAAA,CAAK,CAAA;AAAA,CAAK,EAEPC,CAAAA,CAAiB,GACnB,IAAA,CAAK,YAAA,GAAiB,MACxBA,CAAAA,CAAK,IAAA,CAAK,CAAA,gBAAA,EAAmB,IAAA,CAAK,YAAY,CAAA,CAAE,CAAA,CAElD,IAAMC,CAAAA,CACJD,CAAAA,CAAK,OAAS,CAAA,CAAI;AAAA,QAAA,EAAaA,CAAAA,CAAK,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,CAAM,EAAA,CAEhDE,CAAAA,CACJ,CAAA,2BAAA,EAA8B,IAAA,CAAK,GAAA,CAAIR,CAAAA,CAAM,SAAS,CAAC,CAAA;AAAA,EAAOI,CAAI,CAAA;AAAA,eAAA,EAChDH,CAAAA,CAAGC,CAAE,CAAC,CAAA;AAAA;AAAA,WAAA,EACRD,CAAAA,CAAGC,CAAE,CAAC,CAAA,EACnBK,CAAa,CAAA,CAAA,CAAA,CAElB,MAAM,IAAA,CAAK,QAAA,CAAS,KAAA,CAAM,CAAE,KAAA,CAAOC,CAAI,CAAC,EAC1C,CAEA,MAAM,UAAA,CAAWd,CAAAA,CAAmBe,CAAAA,CAAqC,CACvE,IAAMR,EAAM3B,CAAAA,EAAe,IAAA,CAAK,OAAA,CAAQ,eAAA,CAAgBA,CAAE,CAAA,CAC1D,IAAA,IAAW6B,CAAAA,IAAKM,CAAAA,CAAS,CACvB,IAAMC,CAAAA,CAAO,CAAA,YAAA,EAAe,IAAA,CAAK,GAAA,CAAIhB,CAAS,CAAC,CAAA,YAAA,EAAeO,EAAGE,CAAAA,CAAE,IAAI,CAAC,CAAA,CAAA,EAAIA,CAAAA,CAAE,OAAO,CAAA,CAAA,CAAA,CACrF,MAAM,KAAK,QAAA,CAAS,KAAA,CAAM,CAAE,KAAA,CAAOO,CAAK,CAAC,EAC3C,CACF,CAEA,MAAM,UAAA,CAAWC,CAAAA,CAA4B,CAC3C,MAAM,IAAA,CAAK,QAAA,CAAS,KAAA,CAAM,CAAE,MAAOA,CAAI,CAAC,EAC1C,CAOA,cAAA,CAAejB,CAAAA,CAAyB,CACtC,IAAMkB,EAAS,IAAA,CAAK,OAAA,CAAQ,GAAA,CAAIlB,CAAS,CAAA,CACzC,GAAIkB,CAAAA,CAAQ,CACV,GAAI,CACFA,CAAAA,CAAO,MAAA,CAAO,KAAA,GAChB,CAAA,KAAQ,CAER,CACA,KAAK,OAAA,CAAQ,MAAA,CAAOlB,CAAS,EAC/B,CACF,CAIA,MAAM,UAAA,CACJA,CAAAA,CACAmB,EACe,CACf,GAAIA,CAAAA,CAAK,MAAA,GAAW,CAAA,CAAG,OAGvB,IAAMC,CAAAA,CAAS,MAAM,IAAA,CAAK,iBAAA,CAAkBpB,CAAS,CAAA,CAC/CqB,EAAMF,CAAAA,CAAK,GAAA,CAAKG,CAAAA,GAAS,CAC7B,GAAG,IAAA,CAAK,YAAA,CAAaA,CAAG,CAAA,CACxB,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyBtC,CAAAA,CACvBsC,EAAIhD,CAAmB,CACzB,CACF,CAAA,CAAE,EACF,MAAM,IAAA,CAAK,eAAA,CAAgB0B,CAAAA,CAAWoB,EAAQC,CAAG,EACnD,CAEA,MAAM,UAAA,CACJrB,CAAAA,CACAmB,CAAAA,CACAI,CAAAA,CACe,CACf,GAAIJ,CAAAA,CAAK,MAAA,GAAW,CAAA,CAAG,OACvB,IAAMC,CAAAA,CAAS,MAAM,KAAK,iBAAA,CAAkBpB,CAAAA,CAAWuB,CAAU,CAAA,CAC3DF,CAAAA,CAAMF,CAAAA,CAAK,GAAA,CAAKG,CAAAA,GAAS,CAC7B,GAAG,IAAA,CAAK,YAAA,CAAaA,CAAG,EACxB,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyBtC,CAAAA,CACvBsC,EAAIhD,CAAmB,CACzB,CACF,CAAA,CAAE,CAAA,CACF,MAAM,IAAA,CAAK,eAAA,CAAgB0B,EAAWoB,CAAAA,CAAQC,CAAG,EACnD,CAEA,MAAM,UAAA,CACJrB,CAAAA,CACAuB,CAAAA,CACAC,CAAAA,CACe,CACf,GAAIA,CAAAA,CAAI,MAAA,GAAW,CAAA,CAAG,OACtB,IAAMJ,CAAAA,CAAS,MAAM,KAAK,iBAAA,CAAkBpB,CAAAA,CAAWuB,CAAU,CAAA,CAM3DF,EAAMG,CAAAA,CAAI,GAAA,CAAK5C,CAAAA,GAAQ,CAC3B,CAAC2C,CAAU,EAAG3C,CAAAA,CACd,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyB,kBAC3B,CAAA,CAAE,EACF,MAAM,IAAA,CAAK,eAAA,CAAgBoB,CAAAA,CAAWoB,CAAAA,CAAQC,CAAG,EACnD,CAIA,IAAY,OAAA,EAAU,CACpB,OAAO,IAAA,CAAK,QAAA,CAAS,OAAA,CAAQ,IAAA,CAAK,SAAS,CAC7C,CAEQ,GAAA,CAAIrB,CAAAA,CAA2B,CACrC,OAAO,CAAA,EAAA,EAAK,IAAA,CAAK,SAAS,CAAA,CAAA,EAAIA,CAAS,CAAA,EAAA,CACzC,CAcA,OAAe,aAAA,CAAcyB,CAAAA,CAAiB,CAC5C,IAAMC,CAAAA,CAAKD,EAAE,OAAA,EAAQ,CACrB,OAAA,CAAQ,MAAA,CAAOC,CAAE,CAAA,CAAI,KAAA,EAAO,QAAA,EAC9B,CAGQ,YAAA,CAAaJ,CAAAA,CAAuD,CAC1E,IAAMK,CAAAA,CAA+B,EAAC,CACtC,IAAA,GAAW,CAACC,CAAAA,CAAGC,CAAC,CAAA,GAAK,MAAA,CAAO,QAAQP,CAAG,CAAA,CACrC,GAAIO,CAAAA,GAAM,OACV,GAAIA,CAAAA,YAAa,IAAA,CAEfF,CAAAA,CAAIC,CAAC,CAAA,CAAI9B,CAAAA,CAAgB,aAAA,CAAc+B,CAAC,CAAA,CAAA,KAAA,GAExC,OAAOA,CAAAA,EAAM,QAAA,EACb/B,CAAAA,CAAgB,gBAAA,CAAiB,IAAA,CAAK+B,CAAC,EACvC,CAGA,IAAMJ,CAAAA,CAAI,IAAI,IAAA,CAAKI,CAAC,CAAA,CACf,MAAA,CAAO,MAAMJ,CAAAA,CAAE,OAAA,EAAS,CAAA,CAG3BE,EAAIC,CAAC,CAAA,CAAIC,CAAAA,CAFTF,CAAAA,CAAIC,CAAC,CAAA,CAAI9B,CAAAA,CAAgB,aAAA,CAAc2B,CAAC,EAI5C,CAAA,KAAW,OAAOI,CAAAA,EAAM,UAAYA,CAAAA,GAAM,IAAA,CAExCF,CAAAA,CAAIC,CAAC,CAAA,CAAI,IAAA,CAAK,SAAA,CAAUC,CAAC,EAChB,OAAOA,CAAAA,EAAM,QAAA,CACtBF,CAAAA,CAAIC,CAAC,CAAA,CAAIC,CAAAA,CAAE,QAAA,GAEXF,CAAAA,CAAIC,CAAC,CAAA,CAAIC,CAAAA,CAGb,OAAOF,CACT,CAEA,MAAc,eAAA,CACZ3B,EACAoB,CAAAA,CACAD,CAAAA,CACe,CACf,MAAM7B,CAAAA,CAAU,SAAY,CAC1B,GAAI,CAEF,MADgB8B,CAAAA,CAAO,UAAA,CAAWD,CAAI,CAAA,CACxB,SAAA,GAChB,CAAA,MAAS/B,EAAK,CAGZ,MAAA,IAAA,CAAK,cAAA,CAAeY,CAAS,CAAA,CACvBZ,CACR,CACF,CAAC,EACH,CAEA,MAAc,iBAAA,CACZY,CAAAA,CACAuB,EACc,CACd,IAAML,CAAAA,CAAS,IAAA,CAAK,QAAQ,GAAA,CAAIlB,CAAS,CAAA,CACzC,GAAIkB,CAAAA,CACF,GAAIK,CAAAA,EAAcL,CAAAA,CAAO,aAAeK,CAAAA,CAEtC,IAAA,CAAK,cAAA,CAAevB,CAAS,OAE7B,OAAOkB,CAAAA,CAAO,MAAA,CAIlB,IAAMY,EAAK/C,CAAAA,EAAc,CACpB,IAAA,CAAK,YAAA,GACR,IAAA,CAAK,YAAA,CAAe,IAAI+C,CAAAA,CAAG,cAAc,YAAA,CAAa,CACpD,SAAA,CAAW,IAAA,CAAK,SAClB,CAAC,CAAA,CAAA,CAGH,GAAM,CAAC5B,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,KAAA,CAAMF,CAAS,CAAA,CAAE,WAAA,GACjD+B,CAAAA,CAAW,CAAE,MAAA,CAAQ7B,CAAAA,CAAS,MAAA,EAAQ,MAAA,EAAU,EAAG,EACpDqB,CAAAA,GAIHA,CAAAA,CADErB,CAAAA,CAAS,gBAAA,EAAkB,UAAA,EAAY,OAAA,GAAU,CAAC,CAAA,EACjB6B,EAAS,MAAA,CAAO,CAAC,CAAA,EAAG,IAAA,CAAA,CAGzD,IAAMC,CAAAA,CACJF,CAAAA,CAAG,KAAA,CAAM,yCAAA,CAA0CC,CAAQ,CAAA,CACvDE,CAAAA,CAAkBH,CAAAA,CAAG,KAAA,CAAM,sCAAA,CAC/BE,CAAAA,CACA,KAAA,CACAF,CAAAA,CAAG,MAAM,cAAA,EAAe,CACxBA,CAAAA,CAAG,KAAA,CAAM,0BACX,CAAA,CAEMI,CAAAA,CAAmB,CAAA,SAAA,EAAY,KAAK,SAAS,CAAA,UAAA,EAAa,IAAA,CAAK,SAAS,CAAA,QAAA,EAAWlC,CAAS,CAAA,CAAA,CAM5FmC,CAAAA,CAAa,MAAM,IAAA,CAAK,YAAA,CAAa,sBAAA,CAAuB,CAChE,SAAUL,CAAAA,CAAG,aAAA,CAAc,aAAA,CAC3B,gBAAA,CAAAI,CACF,CAAC,CAAA,CAEKd,CAAAA,CAAS,IAAIU,CAAAA,CAAG,aAAA,CAAc,UAAA,CAAW,CAC7C,WAAAK,CAAAA,CACA,eAAA,CAAAF,CACF,CAAC,CAAA,CAED,OAAA,IAAA,CAAK,OAAA,CAAQ,GAAA,CAAIjC,EAAW,CAAE,MAAA,CAAAoB,CAAAA,CAAQ,UAAA,CAAYG,CAAAA,EAAc,EAAG,CAAC,CAAA,CAC7DH,CACT,CAGA,MAAM,KAAA,EAAuB,CAC3B,OAAW,CAAE,MAAA,CAAAA,CAAO,CAAA,GAAK,KAAK,OAAA,CAAQ,MAAA,EAAO,CAC3C,GAAI,CACFA,CAAAA,CAAO,KAAA,GACT,MAAQ,CAER,CAGF,GADA,IAAA,CAAK,QAAQ,KAAA,EAAM,CACf,IAAA,CAAK,YAAA,EAAgB,OAAO,IAAA,CAAK,YAAA,CAAa,KAAA,EAAU,UAAA,CAC1D,GAAI,CACF,IAAA,CAAK,YAAA,CAAa,QACpB,CAAA,KAAQ,CAER,CAEJ,CACF,CAAA,CArUatB,CAAAA,CAsLa,gBAAA,CACtB,sCAAA,KAvLSsC,CAAAA,CAANtC","file":"bigquery.js","sourcesContent":["/**\n * Internal constants shared between the worker, queue, schema mapper and\n * SQL adapters.\n */\n\n/**\n * Name of the SQL column that stores the publish-time `version` of each\n * sync event. Used by the worker to discard out-of-order PubSub deliveries\n * (the MERGE only updates rows when the incoming version is strictly\n * greater than the stored one).\n *\n * Two underscores prefix avoids collisions with user-defined fields.\n */\nexport const SYNC_VERSION_COLUMN = \"__sync_version\";\n","/**\n * BigQuery type-name utilities used by the {@link BigQueryAdapter}.\n *\n * - {@link normalizeBigQueryType} canonicalises BigQuery type strings so that\n * `INTEGER`/`INT64`, `FLOAT`/`FLOAT64`, `BOOLEAN`/`BOOL`, etc. compare\n * equal during schema-drift detection.\n * - {@link isBigQueryTypeCompatible} returns whether a desired type can\n * safely keep an existing column of another type — only widenings that\n * BigQuery accepts via `ALTER COLUMN SET DATA TYPE` are allowed.\n */\n\n/**\n * Canonicalise a BigQuery type name returned by `getMetadata().schema`\n * (which may use legacy aliases like `INTEGER` or `FLOAT`) so that it can\n * be compared against the type produced by `BigQueryDialect.mapType()`.\n */\nexport function normalizeBigQueryType(type: string): string {\n const upper = type.toUpperCase();\n switch (upper) {\n case \"INTEGER\":\n return \"INT64\";\n case \"FLOAT\":\n return \"FLOAT64\";\n case \"BOOLEAN\":\n return \"BOOL\";\n default:\n return upper;\n }\n}\n\n/**\n * Whether `desired` is the same as, or a safe widening of, `existing`.\n * The widenings mirror what BigQuery allows via\n * `ALTER COLUMN x SET DATA TYPE …` — see\n * https://cloud.google.com/bigquery/docs/managing-table-schemas#change_column_types\n */\nexport function isBigQueryTypeCompatible(\n existing: string,\n desired: string,\n): boolean {\n const a = normalizeBigQueryType(existing);\n const b = normalizeBigQueryType(desired);\n if (a === b) return true;\n\n const widenings: Record<string, string[]> = {\n INT64: [\"NUMERIC\", \"BIGNUMERIC\", \"FLOAT64\"],\n NUMERIC: [\"BIGNUMERIC\", \"FLOAT64\"],\n DATE: [\"DATETIME\", \"TIMESTAMP\"],\n DATETIME: [\"TIMESTAMP\"],\n };\n return widenings[a]?.includes(b) ?? false;\n}\n","/**\n * BigQuery adapter — streams Firestore changes to BigQuery via the\n * **Storage Write API** in CDC (Change Data Capture) mode.\n *\n * Why CDC over the legacy MERGE approach:\n *\n * - **No DML concurrency limit.** MERGE/DELETE DML is bounded by ≈ 2\n * concurrent statements per table; busy collections triggered\n * `Could not serialize access … due to concurrent update` errors.\n * The Storage Write API has no such bound, so multiple Cloud Function\n * instances can flush in parallel without conflicts.\n * - **Cheaper at scale.** Storage Write is roughly ~50% cheaper than\n * legacy streaming inserts, and free for the first 2 TiB / month.\n * - **Same idempotency model.** Each row carries `_CHANGE_SEQUENCE_NUMBER`\n * built from the existing `__sync_version` column, so out-of-order\n * PubSub deliveries are merged correctly by BigQuery.\n *\n * Requirements:\n *\n * - Destination tables **must** declare a `PRIMARY KEY (...) NOT ENFORCED`\n * constraint and be clustered on that key. {@link BigQueryAdapter.createTable}\n * handles both for tables managed by this library.\n * - Tables should be created with `OPTIONS(max_staleness = INTERVAL …)`\n * so the CDC merge runs in the background instead of on every read\n * — see {@link BigQueryAdapterOptions.maxStaleness}.\n * - Service account needs `bigquery.tables.updateData` (e.g. via the\n * `roles/bigquery.dataEditor` role).\n *\n * @example\n * ```ts\n * import { BigQuery } from \"@google-cloud/bigquery\";\n * import { BigQueryAdapter } from \"@lpdjs/firestore-repo-service/sync/bigquery\";\n *\n * const adapter = new BigQueryAdapter({\n * projectId: \"my-project\",\n * datasetId: \"firestore_sync\",\n * bigquery: new BigQuery({ projectId: \"my-project\" }),\n * maxStaleness: \"INTERVAL 15 MINUTE\",\n * });\n * ```\n */\n\nimport type { BigQuery } from \"@google-cloud/bigquery\";\nimport { SYNC_VERSION_COLUMN } from \"../constants\";\nimport type {\n LogicalType,\n SqlAdapter,\n SqlColumn,\n SqlDialect,\n SqlTableDef,\n} from \"../types\";\nimport { normalizeBigQueryType } from \"./bigquery-types\";\n\n// ---------------------------------------------------------------------------\n// Dialect\n// ---------------------------------------------------------------------------\n\n/** BigQuery SQL dialect mapping. */\nclass BigQueryDialect implements SqlDialect {\n readonly name = \"bigquery\";\n\n mapType(logical: LogicalType): string {\n switch (logical) {\n case \"string\":\n return \"STRING\";\n case \"number\":\n return \"FLOAT64\";\n case \"bigint\":\n return \"INT64\";\n case \"boolean\":\n return \"BOOL\";\n case \"timestamp\":\n return \"TIMESTAMP\";\n case \"json\":\n return \"JSON\";\n case \"text\":\n return \"STRING\";\n }\n }\n\n quoteIdentifier(id: string): string {\n return `\\`${id}\\``;\n }\n}\n\n/** Shared BigQuery dialect singleton. */\nexport const bigqueryDialect: SqlDialect = new BigQueryDialect();\n\n// ---------------------------------------------------------------------------\n// Storage Write API loader (kept loose so `@google-cloud/bigquery-storage`\n// stays an OPTIONAL peer dep)\n// ---------------------------------------------------------------------------\n\ntype StorageNs = {\n managedwriter: {\n WriterClient: new (opts?: any) => any;\n JSONWriter: new (params: { connection: any; protoDescriptor: any }) => any;\n DefaultStream: any;\n };\n adapt: {\n convertBigQuerySchemaToStorageTableSchema(schema: any): any;\n convertStorageSchemaToProto2Descriptor(\n schema: any,\n scope: string,\n ...opts: any[]\n ): any;\n withChangeType(): any;\n withChangeSequenceNumber(): any;\n };\n};\n\nlet storageNsCache: StorageNs | null = null;\nfunction loadStorageNs(): StorageNs {\n if (storageNsCache) return storageNsCache;\n // Lazy require so the library does not pull `@google-cloud/bigquery-storage`\n // unless this adapter is actually instantiated.\n // eslint-disable-next-line @typescript-eslint/no-require-imports\n const mod = require(\"@google-cloud/bigquery-storage\");\n storageNsCache = mod as StorageNs;\n return storageNsCache;\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\n\n/**\n * Format `__sync_version` (Firestore commit timestamp in microseconds since\n * epoch) as the 16-character hex string required by `_CHANGE_SEQUENCE_NUMBER`.\n *\n * BigQuery compares sequence numbers lexicographically, so a fixed-width hex\n * representation is mandatory for correct ordering.\n */\nfunction formatChangeSequenceNumber(version: unknown): string {\n let n: bigint;\n if (typeof version === \"bigint\") n = version;\n else if (typeof version === \"number\") n = BigInt(version);\n else if (typeof version === \"string\") n = BigInt(version);\n else n = 0n;\n if (n < 0n) n = 0n;\n return n.toString(16).padStart(16, \"0\");\n}\n\n/**\n * Returns true when the Storage Write append failed with a transient gRPC\n * status (UNAVAILABLE, DEADLINE_EXCEEDED, INTERNAL, ABORTED). Caller can\n * safely retry these.\n */\nfunction isRetryableStorageError(err: unknown): boolean {\n if (!err || typeof err !== \"object\") return false;\n const e = err as { code?: number };\n // gRPC codes: 4=DEADLINE_EXCEEDED, 10=ABORTED, 13=INTERNAL, 14=UNAVAILABLE\n return e.code === 4 || e.code === 10 || e.code === 13 || e.code === 14;\n}\n\nasync function withRetry<T>(\n fn: () => Promise<T>,\n maxRetries = 6,\n baseMs = 200,\n): Promise<T> {\n let attempt = 0;\n // eslint-disable-next-line no-constant-condition\n while (true) {\n try {\n return await fn();\n } catch (err) {\n attempt++;\n if (!isRetryableStorageError(err) || attempt > maxRetries) throw err;\n const cap = baseMs * Math.pow(2, attempt);\n const delay = Math.random() * cap;\n await new Promise((res) => setTimeout(res, delay));\n }\n }\n}\n\n// ---------------------------------------------------------------------------\n// Adapter\n// ---------------------------------------------------------------------------\n\nexport interface BigQueryAdapterOptions {\n /** GCP project id that owns the dataset. */\n projectId: string;\n /** BigQuery dataset id. */\n datasetId: string;\n /**\n * BigQuery client used for DDL operations (createTable, addColumns,\n * getMetadata, executeRaw). Storage Write only handles inserts.\n *\n * Typed as `BigQuery` from `@google-cloud/bigquery` (peer dependency —\n * the type-only import is erased at compile time so the runtime stays\n * free of that module unless the consumer instantiates a client).\n */\n bigquery: BigQuery;\n /**\n * Optional pre-built Storage Write `WriterClient`. When omitted a new\n * one is created lazily on first use.\n */\n writerClient?: any;\n /**\n * Value passed to `OPTIONS(max_staleness = ...)` on `createTable`.\n *\n * **Why this matters**: in BigQuery CDC mode, Storage Write API rows\n * land in a delta buffer. They become visible to queries only after a\n * MERGE applies them to the base table. Two paths exist:\n *\n * - **Background MERGE** — runs periodically in the background, paid\n * for as part of CDC pricing, and never blocks your readers.\n * - **Read-time MERGE** — every `SELECT` query merges the delta on the\n * fly before returning. Always-fresh reads but every query pays for\n * the merge work.\n *\n * `max_staleness` is the tolerated lag between a write and what reads\n * see. Setting it tells BigQuery: \"background MERGE is fine if reads\n * are at most this stale\". When the threshold is exceeded, the next\n * read triggers a one-shot merge.\n *\n * **The default `INTERVAL 0` means \"always read-time merge\"** — every\n * query forces a full merge of pending CDC writes. That makes reads\n * dramatically slower and more expensive on busy tables, so this\n * library defaults to a 15 minute window in production.\n *\n * Recommended:\n * - `INTERVAL 15 MINUTE` in production (good cost/freshness trade-off).\n * - `INTERVAL 1 MINUTE` in dev for quick visibility.\n * - Set to `null` to omit the option entirely (only do this when you\n * know what you are doing — you'll likely hit slow & expensive reads).\n *\n * @default \"INTERVAL 15 MINUTE\"\n */\n maxStaleness?: string | null;\n}\n\n/**\n * BigQuery implementation of {@link SqlAdapter} using the Storage Write API\n * in CDC mode. See module-level docstring for the rationale.\n */\nexport class BigQueryAdapter implements SqlAdapter {\n private readonly bigquery: BigQuery;\n private readonly projectId: string;\n private readonly datasetId: string;\n private readonly maxStaleness: string | null;\n private writerClient: any;\n /** Cache of `{ writer, primaryKey }` per table name. */\n private readonly writers = new Map<\n string,\n { writer: any; primaryKey: string }\n >();\n\n constructor(options: BigQueryAdapterOptions) {\n this.bigquery = options.bigquery;\n this.projectId = options.projectId;\n this.datasetId = options.datasetId;\n this.maxStaleness =\n options.maxStaleness === undefined\n ? \"INTERVAL 15 MINUTE\"\n : options.maxStaleness;\n this.writerClient = options.writerClient;\n }\n\n get dialect(): SqlDialect {\n return bigqueryDialect;\n }\n\n // -------- DDL (delegated to @google-cloud/bigquery) ----------------------\n\n async tableExists(tableName: string): Promise<boolean> {\n const [exists] = await this.dataset.table(tableName).exists();\n return exists;\n }\n\n async getTableColumns(tableName: string): Promise<string[]> {\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const fields: Array<{ name: string }> = metadata.schema?.fields ?? [];\n return fields.map((f) => f.name);\n }\n\n async getTableColumnsWithTypes(\n tableName: string,\n ): Promise<Map<string, string>> {\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const fields: Array<{ name: string; type: string }> =\n metadata.schema?.fields ?? [];\n const result = new Map<string, string>();\n for (const f of fields) {\n result.set(f.name, normalizeBigQueryType(f.type));\n }\n return result;\n }\n\n async createTable(table: SqlTableDef): Promise<void> {\n const qi = (id: string) => this.dialect.quoteIdentifier(id);\n const pk = table.columns.find((c) => c.isPrimaryKey)?.name;\n if (!pk) {\n throw new Error(\n `BigQueryAdapter requires a primary key on table \\`${table.tableName}\\` ` +\n `(Storage Write CDC mode needs PRIMARY KEY NOT ENFORCED).`,\n );\n }\n const cols = table.columns\n .map((c) => {\n const notNull = c.isPrimaryKey ? \" NOT NULL\" : \"\";\n return ` ${qi(c.name)} ${c.sqlType}${notNull}`;\n })\n .join(\",\\n\");\n\n const opts: string[] = [];\n if (this.maxStaleness !== null) {\n opts.push(`max_staleness = ${this.maxStaleness}`);\n }\n const optionsClause =\n opts.length > 0 ? `\\nOPTIONS(${opts.join(\", \")})` : \"\";\n\n const ddl =\n `CREATE TABLE IF NOT EXISTS ${this.fqn(table.tableName)} (\\n${cols},\\n` +\n ` PRIMARY KEY (${qi(pk)}) NOT ENFORCED\\n)` +\n `\\nCLUSTER BY ${qi(pk)}` +\n `${optionsClause};`;\n\n await this.bigquery.query({ query: ddl });\n }\n\n async addColumns(tableName: string, columns: SqlColumn[]): Promise<void> {\n const qi = (id: string) => this.dialect.quoteIdentifier(id);\n for (const c of columns) {\n const stmt = `ALTER TABLE ${this.fqn(tableName)} ADD COLUMN ${qi(c.name)} ${c.sqlType};`;\n await this.bigquery.query({ query: stmt });\n }\n }\n\n async executeRaw(sql: string): Promise<void> {\n await this.bigquery.query({ query: sql });\n }\n\n /**\n * Invalidate the cached writer for a table. Called by the worker after\n * `addColumns` so the next append rebuilds the proto descriptor against\n * the new schema.\n */\n onSchemaChange(tableName: string): void {\n const cached = this.writers.get(tableName);\n if (cached) {\n try {\n cached.writer.close();\n } catch {\n // ignore — connection may already be torn down\n }\n this.writers.delete(tableName);\n }\n }\n\n // -------- Inserts (Storage Write API) -----------------------------------\n\n async insertRows(\n tableName: string,\n rows: Record<string, unknown>[],\n ): Promise<void> {\n if (rows.length === 0) return;\n // Plain inserts have no PK matching, but in CDC mode every row needs a\n // _CHANGE_TYPE. We treat them as UPSERT.\n const writer = await this.getOrCreateWriter(tableName);\n const cdc = rows.map((row) => ({\n ...this.normalizeRow(row),\n _CHANGE_TYPE: \"UPSERT\",\n _CHANGE_SEQUENCE_NUMBER: formatChangeSequenceNumber(\n row[SYNC_VERSION_COLUMN],\n ),\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n async upsertRows(\n tableName: string,\n rows: Record<string, unknown>[],\n primaryKey: string,\n ): Promise<void> {\n if (rows.length === 0) return;\n const writer = await this.getOrCreateWriter(tableName, primaryKey);\n const cdc = rows.map((row) => ({\n ...this.normalizeRow(row),\n _CHANGE_TYPE: \"UPSERT\",\n _CHANGE_SEQUENCE_NUMBER: formatChangeSequenceNumber(\n row[SYNC_VERSION_COLUMN],\n ),\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n async deleteRows(\n tableName: string,\n primaryKey: string,\n ids: string[],\n ): Promise<void> {\n if (ids.length === 0) return;\n const writer = await this.getOrCreateWriter(tableName, primaryKey);\n // Storage Write CDC requires a value for every NOT-NULL column (the PK)\n // and a `_CHANGE_TYPE`. Other columns may be omitted; missing values are\n // interpreted as NULL. We pass the maximum sequence number so the DELETE\n // wins over any concurrent UPSERT for the same key — tombstones from the\n // queue carry no `__sync_version` of their own.\n const cdc = ids.map((id) => ({\n [primaryKey]: id,\n _CHANGE_TYPE: \"DELETE\",\n _CHANGE_SEQUENCE_NUMBER: \"ffffffffffffffff\",\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n // -------- Internal helpers ----------------------------------------------\n\n private get dataset() {\n return this.bigquery.dataset(this.datasetId);\n }\n\n private fqn(tableName: string): string {\n return `\\`${this.datasetId}.${tableName}\\``;\n }\n\n /** ISO 8601 timestamp pattern (e.g. 2026-03-29T20:59:27.394Z) */\n private static readonly ISO_TIMESTAMP_RE =\n /^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}/;\n\n /**\n * Convert a Date or epoch-millis number into the epoch-microseconds string\n * required by the Storage Write API for TIMESTAMP columns.\n *\n * The JSONWriter encodes through protobuf and a TIMESTAMP field is an\n * `int64`. Passing an ISO string would make `Long.fromString` throw\n * `interior hyphen` on the `-` characters.\n */\n private static toEpochMicros(d: Date): string {\n const ms = d.getTime();\n return (BigInt(ms) * 1000n).toString();\n }\n\n /** Convert JS values into shapes accepted by JSONWriter. */\n private normalizeRow(row: Record<string, unknown>): Record<string, unknown> {\n const out: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(row)) {\n if (v === undefined) continue;\n if (v instanceof Date) {\n // TIMESTAMP column → int64 epoch micros (as string to preserve precision)\n out[k] = BigQueryAdapter.toEpochMicros(v);\n } else if (\n typeof v === \"string\" &&\n BigQueryAdapter.ISO_TIMESTAMP_RE.test(v)\n ) {\n // ISO timestamp produced upstream by `serializeDocument` (Firestore\n // Timestamp → ISO string) — convert to epoch micros for protobuf.\n const d = new Date(v);\n if (!Number.isNaN(d.getTime())) {\n out[k] = BigQueryAdapter.toEpochMicros(d);\n } else {\n out[k] = v;\n }\n } else if (typeof v === \"object\" && v !== null) {\n // JSON columns: serialise nested objects/arrays\n out[k] = JSON.stringify(v);\n } else if (typeof v === \"bigint\") {\n out[k] = v.toString();\n } else {\n out[k] = v;\n }\n }\n return out;\n }\n\n private async appendWithRetry(\n tableName: string,\n writer: any,\n rows: Record<string, unknown>[],\n ): Promise<void> {\n await withRetry(async () => {\n try {\n const pending = writer.appendRows(rows);\n await pending.getResult();\n } catch (err) {\n // On certain errors (schema drift, broken connection) drop the\n // cached writer so the next call rebuilds it.\n this.onSchemaChange(tableName);\n throw err;\n }\n });\n }\n\n private async getOrCreateWriter(\n tableName: string,\n primaryKey?: string,\n ): Promise<any> {\n const cached = this.writers.get(tableName);\n if (cached) {\n if (primaryKey && cached.primaryKey !== primaryKey) {\n // PK changed — invalidate and rebuild\n this.onSchemaChange(tableName);\n } else {\n return cached.writer;\n }\n }\n\n const ns = loadStorageNs();\n if (!this.writerClient) {\n this.writerClient = new ns.managedwriter.WriterClient({\n projectId: this.projectId,\n });\n }\n\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const bqSchema = { fields: metadata.schema?.fields ?? [] };\n if (!primaryKey) {\n // Try to recover PK from table constraints if available\n const tableConstraintsPK =\n metadata.tableConstraints?.primaryKey?.columns?.[0];\n primaryKey = tableConstraintsPK ?? bqSchema.fields[0]?.name;\n }\n\n const storageSchema =\n ns.adapt.convertBigQuerySchemaToStorageTableSchema(bqSchema);\n const protoDescriptor = ns.adapt.convertStorageSchemaToProto2Descriptor(\n storageSchema,\n \"Row\",\n ns.adapt.withChangeType(),\n ns.adapt.withChangeSequenceNumber(),\n );\n\n const destinationTable = `projects/${this.projectId}/datasets/${this.datasetId}/tables/${tableName}`;\n // Pass `DefaultStream` as `streamId` (not `streamType`): the `_default`\n // stream is implicit on every table, so the client must short-circuit to\n // `${destinationTable}/streams/_default` instead of calling\n // `createWriteStream({ type: DEFAULT })` which BigQuery rejects with\n // `Unable to create a stream with type TYPE_UNSPECIFIED`.\n const connection = await this.writerClient.createStreamConnection({\n streamId: ns.managedwriter.DefaultStream,\n destinationTable,\n });\n\n const writer = new ns.managedwriter.JSONWriter({\n connection,\n protoDescriptor,\n });\n\n this.writers.set(tableName, { writer, primaryKey: primaryKey ?? \"\" });\n return writer;\n }\n\n /** Close all open writer connections. Useful for graceful shutdown. */\n async close(): Promise<void> {\n for (const { writer } of this.writers.values()) {\n try {\n writer.close();\n } catch {\n /* ignore */\n }\n }\n this.writers.clear();\n if (this.writerClient && typeof this.writerClient.close === \"function\") {\n try {\n this.writerClient.close();\n } catch {\n /* ignore */\n }\n }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"sources":["../../src/sync/constants.ts","../../src/sync/adapters/bigquery-types.ts","../../src/sync/adapters/bigquery.ts"],"names":["SYNC_VERSION_COLUMN","normalizeBigQueryType","type","upper","BigQueryDialect","logical","id","bigqueryDialect","storageNsCache","loadStorageNs","formatChangeSequenceNumber","version","n","isRetryableStorageError","err","e","withRetry","fn","maxRetries","baseMs","attempt","cap","delay","res","_BigQueryAdapter","options","tableName","exists","metadata","f","fields","result","table","qi","pk","c","cols","notNull","opts","optionsClause","ddl","columns","stmt","sql","cached","rows","writer","cdc","row","primaryKey","ids","d","ms","out","k","v","ns","bqSchema","storageSchema","protoDescriptor","destinationTable","connection","BigQueryAdapter"],"mappings":"yPAaO,IAAMA,CAAAA,CAAsB,iBCG5B,SAASC,CAAAA,CAAsBC,EAAsB,CAC1D,IAAMC,EAAQD,CAAAA,CAAK,WAAA,GACnB,OAAQC,CAAAA,EACN,KAAK,SAAA,CACH,OAAO,OAAA,CACT,KAAK,OAAA,CACH,OAAO,SAAA,CACT,KAAK,UACH,OAAO,MAAA,CACT,QACE,OAAOA,CACX,CACF,CC8BA,IAAMC,CAAAA,CAAN,KAA4C,CAA5C,WAAA,EAAA,CACE,KAAS,IAAA,CAAO,WAAA,CAEhB,QAAQC,CAAAA,CAA8B,CACpC,OAAQA,CAAAA,EACN,KAAK,QAAA,CACH,OAAO,QAAA,CACT,KAAK,QAAA,CACH,OAAO,UACT,KAAK,QAAA,CACH,OAAO,OAAA,CACT,KAAK,SAAA,CACH,OAAO,MAAA,CACT,KAAK,YACH,OAAO,WAAA,CACT,KAAK,MAAA,CACH,OAAO,OACT,KAAK,MAAA,CACH,OAAO,QACX,CACF,CAEA,gBAAgBC,CAAAA,CAAoB,CAClC,OAAO,CAAA,EAAA,EAAKA,CAAE,IAChB,CACF,CAAA,CAGaC,CAAAA,CAA8B,IAAIH,CAAAA,CAyB3CI,CAAAA,CAAmC,KACvC,SAASC,CAAAA,EAA2B,CAClC,OAAID,CAAAA,GAKJA,EADY,CAAA,CAAQ,gCAAgC,CAAA,CAE7CA,CAAAA,CACT,CAaA,SAASE,EAA2BC,CAAAA,CAA0B,CAC5D,IAAIC,CAAAA,CACJ,OAAI,OAAOD,CAAAA,EAAY,QAAA,CAAUC,CAAAA,CAAID,CAAAA,CAC5B,OAAOA,CAAAA,EAAY,UACnB,OAAOA,CAAAA,EAAY,SADUC,CAAAA,CAAI,MAAA,CAAOD,CAAO,CAAA,CAEnDC,CAAAA,CAAI,EAAA,CACLA,CAAAA,CAAI,EAAA,GAAIA,CAAAA,CAAI,IACTA,CAAAA,CAAE,QAAA,CAAS,EAAE,CAAA,CAAE,QAAA,CAAS,GAAI,GAAG,CACxC,CAOA,SAASC,CAAAA,CAAwBC,CAAAA,CAAuB,CACtD,GAAI,CAACA,GAAO,OAAOA,CAAAA,EAAQ,SAAU,OAAO,MAAA,CAC5C,IAAMC,CAAAA,CAAID,CAAAA,CAEV,OAAOC,EAAE,IAAA,GAAS,CAAA,EAAKA,EAAE,IAAA,GAAS,EAAA,EAAMA,EAAE,IAAA,GAAS,EAAA,EAAMA,CAAAA,CAAE,IAAA,GAAS,EACtE,CAEA,eAAeC,CAAAA,CACbC,CAAAA,CACAC,EAAa,CAAA,CACbC,CAAAA,CAAS,IACG,CACZ,IAAIC,CAAAA,CAAU,CAAA,CAEd,OACE,GAAI,CACF,OAAO,MAAMH,GACf,CAAA,MAASH,EAAK,CAEZ,GADAM,CAAAA,EAAAA,CACI,CAACP,CAAAA,CAAwBC,CAAG,GAAKM,CAAAA,CAAUF,CAAAA,CAAY,MAAMJ,CAAAA,CACjE,IAAMO,EAAMF,CAAAA,CAAS,IAAA,CAAK,GAAA,CAAI,CAAA,CAAGC,CAAO,CAAA,CAClCE,EAAQ,IAAA,CAAK,MAAA,GAAWD,CAAAA,CAC9B,MAAM,IAAI,OAAA,CAASE,CAAAA,EAAQ,WAAWA,CAAAA,CAAKD,CAAK,CAAC,EACnD,CAEJ,CA0FO,IAAME,CAAAA,CAAN,MAAMA,CAAsC,CAYjD,WAAA,CAAYC,CAAAA,CAAiC,CAL7C,IAAA,CAAiB,QAAU,IAAI,GAAA,CAM7B,KAAK,QAAA,CAAWA,CAAAA,CAAQ,SACxB,IAAA,CAAK,SAAA,CAAYA,CAAAA,CAAQ,SAAA,CACzB,IAAA,CAAK,SAAA,CAAYA,EAAQ,SAAA,CACzB,IAAA,CAAK,aACHA,CAAAA,CAAQ,YAAA,GAAiB,OACrB,oBAAA,CACAA,CAAAA,CAAQ,YAAA,CACd,IAAA,CAAK,YAAA,CAAeA,CAAAA,CAAQ,aAC9B,CAEA,IAAI,SAAsB,CACxB,OAAOlB,CACT,CAIA,MAAM,WAAA,CAAYmB,CAAAA,CAAqC,CACrD,GAAM,CAACC,CAAM,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAMD,CAAS,CAAA,CAAE,MAAA,EAAO,CAC5D,OAAOC,CACT,CAEA,MAAM,eAAA,CAAgBD,EAAsC,CAC1D,GAAM,CAACE,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,KAAA,CAAMF,CAAS,CAAA,CAAE,WAAA,GAEvD,OAAA,CADwCE,CAAAA,CAAS,QAAQ,MAAA,EAAU,EAAC,EACtD,GAAA,CAAKC,CAAAA,EAAMA,CAAAA,CAAE,IAAI,CACjC,CAEA,MAAM,wBAAA,CACJH,CAAAA,CAC8B,CAC9B,GAAM,CAACE,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,QAAQ,KAAA,CAAMF,CAAS,EAAE,WAAA,EAAY,CAC7DI,EACJF,CAAAA,CAAS,MAAA,EAAQ,MAAA,EAAU,EAAC,CACxBG,CAAAA,CAAS,IAAI,GAAA,CACnB,IAAA,IAAWF,KAAKC,CAAAA,CACdC,CAAAA,CAAO,IAAIF,CAAAA,CAAE,IAAA,CAAM5B,CAAAA,CAAsB4B,CAAAA,CAAE,IAAI,CAAC,EAElD,OAAOE,CACT,CAEA,MAAM,WAAA,CAAYC,EAAmC,CACnD,IAAMC,CAAAA,CAAM3B,CAAAA,EAAe,IAAA,CAAK,OAAA,CAAQ,gBAAgBA,CAAE,CAAA,CACpD4B,EAAKF,CAAAA,CAAM,OAAA,CAAQ,KAAMG,CAAAA,EAAMA,CAAAA,CAAE,YAAY,CAAA,EAAG,IAAA,CACtD,GAAI,CAACD,CAAAA,CACH,MAAM,IAAI,KAAA,CACR,CAAA,kDAAA,EAAqDF,EAAM,SAAS,CAAA,2DAAA,CAEtE,CAAA,CAEF,IAAMI,CAAAA,CAAOJ,CAAAA,CAAM,QAChB,GAAA,CAAKG,CAAAA,EAAM,CACV,IAAME,CAAAA,CAAUF,EAAE,YAAA,CAAe,WAAA,CAAc,EAAA,CAC/C,OAAO,CAAA,EAAA,EAAKF,CAAAA,CAAGE,EAAE,IAAI,CAAC,IAAIA,CAAAA,CAAE,OAAO,GAAGE,CAAO,CAAA,CAC/C,CAAC,CAAA,CACA,IAAA,CAAK,CAAA;AAAA,CAAK,EAEPC,CAAAA,CAAiB,GACnB,IAAA,CAAK,YAAA,GAAiB,MACxBA,CAAAA,CAAK,IAAA,CAAK,CAAA,gBAAA,EAAmB,IAAA,CAAK,YAAY,CAAA,CAAE,CAAA,CAElD,IAAMC,CAAAA,CACJD,CAAAA,CAAK,OAAS,CAAA,CAAI;AAAA,QAAA,EAAaA,CAAAA,CAAK,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,CAAM,EAAA,CAEhDE,CAAAA,CACJ,CAAA,2BAAA,EAA8B,IAAA,CAAK,GAAA,CAAIR,CAAAA,CAAM,SAAS,CAAC,CAAA;AAAA,EAAOI,CAAI,CAAA;AAAA,eAAA,EAChDH,CAAAA,CAAGC,CAAE,CAAC,CAAA;AAAA;AAAA,WAAA,EACRD,CAAAA,CAAGC,CAAE,CAAC,CAAA,EACnBK,CAAa,CAAA,CAAA,CAAA,CAElB,MAAM,IAAA,CAAK,QAAA,CAAS,KAAA,CAAM,CAAE,KAAA,CAAOC,CAAI,CAAC,EAC1C,CAEA,MAAM,UAAA,CAAWd,CAAAA,CAAmBe,CAAAA,CAAqC,CACvE,IAAMR,EAAM3B,CAAAA,EAAe,IAAA,CAAK,OAAA,CAAQ,eAAA,CAAgBA,CAAE,CAAA,CAC1D,IAAA,IAAW6B,CAAAA,IAAKM,CAAAA,CAAS,CACvB,IAAMC,CAAAA,CAAO,CAAA,YAAA,EAAe,IAAA,CAAK,GAAA,CAAIhB,CAAS,CAAC,CAAA,YAAA,EAAeO,EAAGE,CAAAA,CAAE,IAAI,CAAC,CAAA,CAAA,EAAIA,CAAAA,CAAE,OAAO,CAAA,CAAA,CAAA,CACrF,MAAM,KAAK,QAAA,CAAS,KAAA,CAAM,CAAE,KAAA,CAAOO,CAAK,CAAC,EAC3C,CACF,CAEA,MAAM,UAAA,CAAWC,CAAAA,CAA4B,CAC3C,MAAM,IAAA,CAAK,QAAA,CAAS,KAAA,CAAM,CAAE,MAAOA,CAAI,CAAC,EAC1C,CAOA,cAAA,CAAejB,CAAAA,CAAyB,CACtC,IAAMkB,EAAS,IAAA,CAAK,OAAA,CAAQ,GAAA,CAAIlB,CAAS,CAAA,CACzC,GAAIkB,CAAAA,CAAQ,CACV,GAAI,CACFA,CAAAA,CAAO,MAAA,CAAO,KAAA,GAChB,CAAA,KAAQ,CAER,CACA,KAAK,OAAA,CAAQ,MAAA,CAAOlB,CAAS,EAC/B,CACF,CAIA,MAAM,UAAA,CACJA,CAAAA,CACAmB,EACe,CACf,GAAIA,CAAAA,CAAK,MAAA,GAAW,CAAA,CAAG,OAGvB,IAAMC,CAAAA,CAAS,MAAM,IAAA,CAAK,iBAAA,CAAkBpB,CAAS,CAAA,CAC/CqB,EAAMF,CAAAA,CAAK,GAAA,CAAKG,CAAAA,GAAS,CAC7B,GAAG,IAAA,CAAK,YAAA,CAAaA,CAAG,CAAA,CACxB,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyBtC,CAAAA,CACvBsC,EAAIhD,CAAmB,CACzB,CACF,CAAA,CAAE,EACF,MAAM,IAAA,CAAK,eAAA,CAAgB0B,CAAAA,CAAWoB,EAAQC,CAAG,EACnD,CAEA,MAAM,UAAA,CACJrB,CAAAA,CACAmB,CAAAA,CACAI,CAAAA,CACe,CACf,GAAIJ,CAAAA,CAAK,MAAA,GAAW,CAAA,CAAG,OACvB,IAAMC,CAAAA,CAAS,MAAM,KAAK,iBAAA,CAAkBpB,CAAAA,CAAWuB,CAAU,CAAA,CAC3DF,CAAAA,CAAMF,CAAAA,CAAK,GAAA,CAAKG,CAAAA,GAAS,CAC7B,GAAG,IAAA,CAAK,YAAA,CAAaA,CAAG,EACxB,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyBtC,CAAAA,CACvBsC,EAAIhD,CAAmB,CACzB,CACF,CAAA,CAAE,CAAA,CACF,MAAM,IAAA,CAAK,eAAA,CAAgB0B,EAAWoB,CAAAA,CAAQC,CAAG,EACnD,CAEA,MAAM,UAAA,CACJrB,CAAAA,CACAuB,CAAAA,CACAC,CAAAA,CACe,CACf,GAAIA,CAAAA,CAAI,MAAA,GAAW,CAAA,CAAG,OACtB,IAAMJ,CAAAA,CAAS,MAAM,KAAK,iBAAA,CAAkBpB,CAAAA,CAAWuB,CAAU,CAAA,CAM3DF,EAAMG,CAAAA,CAAI,GAAA,CAAK5C,CAAAA,GAAQ,CAC3B,CAAC2C,CAAU,EAAG3C,CAAAA,CACd,YAAA,CAAc,QAAA,CACd,uBAAA,CAAyB,kBAC3B,CAAA,CAAE,EACF,MAAM,IAAA,CAAK,eAAA,CAAgBoB,CAAAA,CAAWoB,CAAAA,CAAQC,CAAG,EACnD,CAIA,IAAY,OAAA,EAAU,CACpB,OAAO,IAAA,CAAK,QAAA,CAAS,OAAA,CAAQ,IAAA,CAAK,SAAS,CAC7C,CAEQ,GAAA,CAAIrB,CAAAA,CAA2B,CACrC,OAAO,CAAA,EAAA,EAAK,IAAA,CAAK,SAAS,CAAA,CAAA,EAAIA,CAAS,CAAA,EAAA,CACzC,CAcA,OAAe,aAAA,CAAcyB,CAAAA,CAAiB,CAC5C,IAAMC,CAAAA,CAAKD,EAAE,OAAA,EAAQ,CACrB,OAAA,CAAQ,MAAA,CAAOC,CAAE,CAAA,CAAI,KAAA,EAAO,QAAA,EAC9B,CAGQ,YAAA,CAAaJ,CAAAA,CAAuD,CAC1E,IAAMK,CAAAA,CAA+B,EAAC,CACtC,IAAA,GAAW,CAACC,CAAAA,CAAGC,CAAC,CAAA,GAAK,MAAA,CAAO,QAAQP,CAAG,CAAA,CACrC,GAAIO,CAAAA,GAAM,OACV,GAAIA,CAAAA,YAAa,IAAA,CAEfF,CAAAA,CAAIC,CAAC,CAAA,CAAI9B,CAAAA,CAAgB,aAAA,CAAc+B,CAAC,CAAA,CAAA,KAAA,GAExC,OAAOA,CAAAA,EAAM,QAAA,EACb/B,CAAAA,CAAgB,gBAAA,CAAiB,IAAA,CAAK+B,CAAC,EACvC,CAGA,IAAMJ,CAAAA,CAAI,IAAI,IAAA,CAAKI,CAAC,CAAA,CACf,MAAA,CAAO,MAAMJ,CAAAA,CAAE,OAAA,EAAS,CAAA,CAG3BE,EAAIC,CAAC,CAAA,CAAIC,CAAAA,CAFTF,CAAAA,CAAIC,CAAC,CAAA,CAAI9B,CAAAA,CAAgB,aAAA,CAAc2B,CAAC,EAI5C,CAAA,KAAW,OAAOI,CAAAA,EAAM,UAAYA,CAAAA,GAAM,IAAA,CAExCF,CAAAA,CAAIC,CAAC,CAAA,CAAI,IAAA,CAAK,SAAA,CAAUC,CAAC,EAChB,OAAOA,CAAAA,EAAM,QAAA,CACtBF,CAAAA,CAAIC,CAAC,CAAA,CAAIC,CAAAA,CAAE,QAAA,GAEXF,CAAAA,CAAIC,CAAC,CAAA,CAAIC,CAAAA,CAGb,OAAOF,CACT,CAEA,MAAc,eAAA,CACZ3B,EACAoB,CAAAA,CACAD,CAAAA,CACe,CACf,MAAM7B,CAAAA,CAAU,SAAY,CAC1B,GAAI,CAEF,MADgB8B,CAAAA,CAAO,UAAA,CAAWD,CAAI,CAAA,CACxB,SAAA,GAChB,CAAA,MAAS/B,EAAK,CAGZ,MAAA,IAAA,CAAK,cAAA,CAAeY,CAAS,CAAA,CACvBZ,CACR,CACF,CAAC,EACH,CAEA,MAAc,iBAAA,CACZY,CAAAA,CACAuB,EACc,CACd,IAAML,CAAAA,CAAS,IAAA,CAAK,QAAQ,GAAA,CAAIlB,CAAS,CAAA,CACzC,GAAIkB,CAAAA,CACF,GAAIK,CAAAA,EAAcL,CAAAA,CAAO,aAAeK,CAAAA,CAEtC,IAAA,CAAK,cAAA,CAAevB,CAAS,OAE7B,OAAOkB,CAAAA,CAAO,MAAA,CAIlB,IAAMY,EAAK/C,CAAAA,EAAc,CACpB,IAAA,CAAK,YAAA,GACR,IAAA,CAAK,YAAA,CAAe,IAAI+C,CAAAA,CAAG,cAAc,YAAA,CAAa,CACpD,SAAA,CAAW,IAAA,CAAK,SAClB,CAAC,CAAA,CAAA,CAGH,GAAM,CAAC5B,CAAQ,CAAA,CAAI,MAAM,IAAA,CAAK,OAAA,CAAQ,KAAA,CAAMF,CAAS,CAAA,CAAE,WAAA,GACjD+B,CAAAA,CAAW,CAAE,MAAA,CAAQ7B,CAAAA,CAAS,MAAA,EAAQ,MAAA,EAAU,EAAG,EACpDqB,CAAAA,GAIHA,CAAAA,CADErB,CAAAA,CAAS,gBAAA,EAAkB,UAAA,EAAY,OAAA,GAAU,CAAC,CAAA,EACjB6B,EAAS,MAAA,CAAO,CAAC,CAAA,EAAG,IAAA,CAAA,CAGzD,IAAMC,CAAAA,CACJF,CAAAA,CAAG,KAAA,CAAM,yCAAA,CAA0CC,CAAQ,CAAA,CACvDE,CAAAA,CAAkBH,CAAAA,CAAG,KAAA,CAAM,sCAAA,CAC/BE,CAAAA,CACA,KAAA,CACAF,CAAAA,CAAG,MAAM,cAAA,EAAe,CACxBA,CAAAA,CAAG,KAAA,CAAM,0BACX,CAAA,CAEMI,CAAAA,CAAmB,CAAA,SAAA,EAAY,KAAK,SAAS,CAAA,UAAA,EAAa,IAAA,CAAK,SAAS,CAAA,QAAA,EAAWlC,CAAS,CAAA,CAAA,CAM5FmC,CAAAA,CAAa,MAAM,IAAA,CAAK,YAAA,CAAa,sBAAA,CAAuB,CAChE,SAAUL,CAAAA,CAAG,aAAA,CAAc,aAAA,CAC3B,gBAAA,CAAAI,CACF,CAAC,CAAA,CAEKd,CAAAA,CAAS,IAAIU,CAAAA,CAAG,aAAA,CAAc,UAAA,CAAW,CAC7C,WAAAK,CAAAA,CACA,eAAA,CAAAF,CACF,CAAC,CAAA,CAED,OAAA,IAAA,CAAK,OAAA,CAAQ,GAAA,CAAIjC,EAAW,CAAE,MAAA,CAAAoB,CAAAA,CAAQ,UAAA,CAAYG,CAAAA,EAAc,EAAG,CAAC,CAAA,CAC7DH,CACT,CAGA,MAAM,KAAA,EAAuB,CAC3B,OAAW,CAAE,MAAA,CAAAA,CAAO,CAAA,GAAK,KAAK,OAAA,CAAQ,MAAA,EAAO,CAC3C,GAAI,CACFA,CAAAA,CAAO,KAAA,GACT,MAAQ,CAER,CAGF,GADA,IAAA,CAAK,QAAQ,KAAA,EAAM,CACf,IAAA,CAAK,YAAA,EAAgB,OAAO,IAAA,CAAK,YAAA,CAAa,KAAA,EAAU,UAAA,CAC1D,GAAI,CACF,IAAA,CAAK,YAAA,CAAa,QACpB,CAAA,KAAQ,CAER,CAEJ,CACF,CAAA,CArUatB,CAAAA,CAsLa,gBAAA,CACtB,sCAAA,KAvLSsC,CAAAA,CAANtC","file":"bigquery.js","sourcesContent":["/**\n * Internal constants shared between the worker, queue, schema mapper and\n * SQL adapters.\n */\n\n/**\n * Name of the SQL column that stores the publish-time `version` of each\n * sync event. Used by the worker to discard out-of-order PubSub deliveries\n * (the MERGE only updates rows when the incoming version is strictly\n * greater than the stored one).\n *\n * Two underscores prefix avoids collisions with user-defined fields.\n */\nexport const SYNC_VERSION_COLUMN = \"__sync_version\";\n","/**\n * BigQuery type-name utilities used by the {@link BigQueryAdapter}.\n *\n * - {@link normalizeBigQueryType} canonicalises BigQuery type strings so that\n * `INTEGER`/`INT64`, `FLOAT`/`FLOAT64`, `BOOLEAN`/`BOOL`, etc. compare\n * equal during schema-drift detection.\n * - {@link isBigQueryTypeCompatible} returns whether a desired type can\n * safely keep an existing column of another type — only widenings that\n * BigQuery accepts via `ALTER COLUMN SET DATA TYPE` are allowed.\n */\n\n/**\n * Canonicalise a BigQuery type name returned by `getMetadata().schema`\n * (which may use legacy aliases like `INTEGER` or `FLOAT`) so that it can\n * be compared against the type produced by `BigQueryDialect.mapType()`.\n */\nexport function normalizeBigQueryType(type: string): string {\n const upper = type.toUpperCase();\n switch (upper) {\n case \"INTEGER\":\n return \"INT64\";\n case \"FLOAT\":\n return \"FLOAT64\";\n case \"BOOLEAN\":\n return \"BOOL\";\n default:\n return upper;\n }\n}\n\n/**\n * Whether `desired` is the same as, or a safe widening of, `existing`.\n * The widenings mirror what BigQuery allows via\n * `ALTER COLUMN x SET DATA TYPE …` — see\n * https://cloud.google.com/bigquery/docs/managing-table-schemas#change_column_types\n */\nexport function isBigQueryTypeCompatible(\n existing: string,\n desired: string,\n): boolean {\n const a = normalizeBigQueryType(existing);\n const b = normalizeBigQueryType(desired);\n if (a === b) return true;\n\n const widenings: Record<string, string[]> = {\n INT64: [\"NUMERIC\", \"BIGNUMERIC\", \"FLOAT64\"],\n NUMERIC: [\"BIGNUMERIC\", \"FLOAT64\"],\n DATE: [\"DATETIME\", \"TIMESTAMP\"],\n DATETIME: [\"TIMESTAMP\"],\n };\n return widenings[a]?.includes(b) ?? false;\n}\n","/**\n * BigQuery adapter — streams Firestore changes to BigQuery via the\n * **Storage Write API** in CDC (Change Data Capture) mode.\n *\n * Why CDC over the legacy MERGE approach:\n *\n * - **No DML concurrency limit.** MERGE/DELETE DML is bounded by ≈ 2\n * concurrent statements per table; busy collections triggered\n * `Could not serialize access … due to concurrent update` errors.\n * The Storage Write API has no such bound, so multiple Cloud Function\n * instances can flush in parallel without conflicts.\n * - **Cheaper at scale.** Storage Write is roughly ~50% cheaper than\n * legacy streaming inserts, and free for the first 2 TiB / month.\n * - **Same idempotency model.** Each row carries `_CHANGE_SEQUENCE_NUMBER`\n * built from the existing `__sync_version` column, so out-of-order\n * PubSub deliveries are merged correctly by BigQuery.\n *\n * Requirements:\n *\n * - Destination tables **must** declare a `PRIMARY KEY (...) NOT ENFORCED`\n * constraint and be clustered on that key. {@link BigQueryAdapter.createTable}\n * handles both for tables managed by this library.\n * - Tables should be created with `OPTIONS(max_staleness = INTERVAL …)`\n * so the CDC merge runs in the background instead of on every read\n * — see {@link BigQueryAdapterOptions.maxStaleness}.\n * - Service account needs `bigquery.tables.updateData` (e.g. via the\n * `roles/bigquery.dataEditor` role).\n *\n * @example\n * ```ts\n * import { BigQuery } from \"@google-cloud/bigquery\";\n * import { BigQueryAdapter } from \"@lpdjs/firestore-repo-service/sync/bigquery\";\n *\n * const adapter = new BigQueryAdapter({\n * projectId: \"my-project\",\n * datasetId: \"firestore_sync\",\n * bigquery: new BigQuery({ projectId: \"my-project\" }),\n * maxStaleness: \"INTERVAL 15 MINUTE\",\n * });\n * ```\n */\n\nimport type { BigQuery } from \"@google-cloud/bigquery\";\nimport { SYNC_VERSION_COLUMN } from \"../constants\";\nimport type {\n LogicalType,\n SqlAdapter,\n SqlColumn,\n SqlDialect,\n SqlTableDef,\n} from \"../types\";\nimport { normalizeBigQueryType } from \"./bigquery-types\";\n\n// ---------------------------------------------------------------------------\n// Dialect\n// ---------------------------------------------------------------------------\n\n/** BigQuery SQL dialect mapping. */\nclass BigQueryDialect implements SqlDialect {\n readonly name = \"bigquery\";\n\n mapType(logical: LogicalType): string {\n switch (logical) {\n case \"string\":\n return \"STRING\";\n case \"number\":\n return \"FLOAT64\";\n case \"bigint\":\n return \"INT64\";\n case \"boolean\":\n return \"BOOL\";\n case \"timestamp\":\n return \"TIMESTAMP\";\n case \"json\":\n return \"JSON\";\n case \"text\":\n return \"STRING\";\n }\n }\n\n quoteIdentifier(id: string): string {\n return `\\`${id}\\``;\n }\n}\n\n/** Shared BigQuery dialect singleton. */\nexport const bigqueryDialect: SqlDialect = new BigQueryDialect();\n\n// ---------------------------------------------------------------------------\n// Storage Write API loader (kept loose so `@google-cloud/bigquery-storage`\n// stays an OPTIONAL peer dep)\n// ---------------------------------------------------------------------------\n\ntype StorageNs = {\n managedwriter: {\n WriterClient: new (opts?: any) => any;\n JSONWriter: new (params: { connection: any; protoDescriptor: any }) => any;\n DefaultStream: any;\n };\n adapt: {\n convertBigQuerySchemaToStorageTableSchema(schema: any): any;\n convertStorageSchemaToProto2Descriptor(\n schema: any,\n scope: string,\n ...opts: any[]\n ): any;\n withChangeType(): any;\n withChangeSequenceNumber(): any;\n };\n};\n\nlet storageNsCache: StorageNs | null = null;\nfunction loadStorageNs(): StorageNs {\n if (storageNsCache) return storageNsCache;\n // Lazy require so the library does not pull `@google-cloud/bigquery-storage`\n // unless this adapter is actually instantiated.\n // eslint-disable-next-line @typescript-eslint/no-require-imports\n const mod = require(\"@google-cloud/bigquery-storage\");\n storageNsCache = mod as StorageNs;\n return storageNsCache;\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\n\n/**\n * Format `__sync_version` (Firestore commit timestamp in microseconds since\n * epoch) as the 16-character hex string required by `_CHANGE_SEQUENCE_NUMBER`.\n *\n * BigQuery compares sequence numbers lexicographically, so a fixed-width hex\n * representation is mandatory for correct ordering.\n */\nfunction formatChangeSequenceNumber(version: unknown): string {\n let n: bigint;\n if (typeof version === \"bigint\") n = version;\n else if (typeof version === \"number\") n = BigInt(version);\n else if (typeof version === \"string\") n = BigInt(version);\n else n = 0n;\n if (n < 0n) n = 0n;\n return n.toString(16).padStart(16, \"0\");\n}\n\n/**\n * Returns true when the Storage Write append failed with a transient gRPC\n * status (UNAVAILABLE, DEADLINE_EXCEEDED, INTERNAL, ABORTED). Caller can\n * safely retry these.\n */\nfunction isRetryableStorageError(err: unknown): boolean {\n if (!err || typeof err !== \"object\") return false;\n const e = err as { code?: number };\n // gRPC codes: 4=DEADLINE_EXCEEDED, 10=ABORTED, 13=INTERNAL, 14=UNAVAILABLE\n return e.code === 4 || e.code === 10 || e.code === 13 || e.code === 14;\n}\n\nasync function withRetry<T>(\n fn: () => Promise<T>,\n maxRetries = 6,\n baseMs = 200,\n): Promise<T> {\n let attempt = 0;\n // eslint-disable-next-line no-constant-condition\n while (true) {\n try {\n return await fn();\n } catch (err) {\n attempt++;\n if (!isRetryableStorageError(err) || attempt > maxRetries) throw err;\n const cap = baseMs * Math.pow(2, attempt);\n const delay = Math.random() * cap;\n await new Promise((res) => setTimeout(res, delay));\n }\n }\n}\n\n// ---------------------------------------------------------------------------\n// Adapter\n// ---------------------------------------------------------------------------\n\n/**\n * Minimal structural shape of `@google-cloud/bigquery`'s `BigQuery` client\n * actually used by the adapter (`.dataset(...)` and `.query(...)`).\n *\n * Why structural and not just `BigQuery`: TypeScript treats classes with\n * private fields nominally. When a consumer's `node_modules` ends up with\n * a *second* copy of `@google-cloud/bigquery` (very common in workspace\n * setups, peer-dep dedup misses, monorepo nesting), the `BigQuery` they\n * import is *technically* a different type than the one this library's\n * `BigQuery` references — even though they are runtime-identical — and\n * assignment fails with `Types have separate declarations of a private\n * property '_universeDomain'`.\n *\n * Accepting a structural superset sidesteps the duplicate-install issue\n * without giving up type safety on the methods we actually call.\n */\nexport interface BigQueryLike {\n dataset(datasetId: string): {\n table(tableName: string): any;\n [key: string]: any;\n };\n query(options: { query: string; params?: unknown[] } | string): Promise<any>;\n}\n\nexport interface BigQueryAdapterOptions {\n /** GCP project id that owns the dataset. */\n projectId: string;\n /** BigQuery dataset id. */\n datasetId: string;\n /**\n * BigQuery client used for DDL operations (createTable, addColumns,\n * getMetadata, executeRaw). Storage Write only handles inserts.\n *\n * Typed as the structural {@link BigQueryLike} (a superset satisfied by\n * `BigQuery` from `@google-cloud/bigquery`) to avoid TypeScript nominal\n * mismatches when the consumer's project ends up with two copies of\n * `@google-cloud/bigquery` in different `node_modules`. Pass\n * `new BigQuery({...})` from your own install — it satisfies this shape\n * structurally.\n */\n bigquery: BigQueryLike | BigQuery;\n /**\n * Optional pre-built Storage Write `WriterClient`. When omitted a new\n * one is created lazily on first use.\n */\n writerClient?: any;\n /**\n * Value passed to `OPTIONS(max_staleness = ...)` on `createTable`.\n *\n * **Why this matters**: in BigQuery CDC mode, Storage Write API rows\n * land in a delta buffer. They become visible to queries only after a\n * MERGE applies them to the base table. Two paths exist:\n *\n * - **Background MERGE** — runs periodically in the background, paid\n * for as part of CDC pricing, and never blocks your readers.\n * - **Read-time MERGE** — every `SELECT` query merges the delta on the\n * fly before returning. Always-fresh reads but every query pays for\n * the merge work.\n *\n * `max_staleness` is the tolerated lag between a write and what reads\n * see. Setting it tells BigQuery: \"background MERGE is fine if reads\n * are at most this stale\". When the threshold is exceeded, the next\n * read triggers a one-shot merge.\n *\n * **The default `INTERVAL 0` means \"always read-time merge\"** — every\n * query forces a full merge of pending CDC writes. That makes reads\n * dramatically slower and more expensive on busy tables, so this\n * library defaults to a 15 minute window in production.\n *\n * Recommended:\n * - `INTERVAL 15 MINUTE` in production (good cost/freshness trade-off).\n * - `INTERVAL 1 MINUTE` in dev for quick visibility.\n * - Set to `null` to omit the option entirely (only do this when you\n * know what you are doing — you'll likely hit slow & expensive reads).\n *\n * @default \"INTERVAL 15 MINUTE\"\n */\n maxStaleness?: string | null;\n}\n\n/**\n * BigQuery implementation of {@link SqlAdapter} using the Storage Write API\n * in CDC mode. See module-level docstring for the rationale.\n */\nexport class BigQueryAdapter implements SqlAdapter {\n private readonly bigquery: BigQueryLike;\n private readonly projectId: string;\n private readonly datasetId: string;\n private readonly maxStaleness: string | null;\n private writerClient: any;\n /** Cache of `{ writer, primaryKey }` per table name. */\n private readonly writers = new Map<\n string,\n { writer: any; primaryKey: string }\n >();\n\n constructor(options: BigQueryAdapterOptions) {\n this.bigquery = options.bigquery;\n this.projectId = options.projectId;\n this.datasetId = options.datasetId;\n this.maxStaleness =\n options.maxStaleness === undefined\n ? \"INTERVAL 15 MINUTE\"\n : options.maxStaleness;\n this.writerClient = options.writerClient;\n }\n\n get dialect(): SqlDialect {\n return bigqueryDialect;\n }\n\n // -------- DDL (delegated to @google-cloud/bigquery) ----------------------\n\n async tableExists(tableName: string): Promise<boolean> {\n const [exists] = await this.dataset.table(tableName).exists();\n return exists;\n }\n\n async getTableColumns(tableName: string): Promise<string[]> {\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const fields: Array<{ name: string }> = metadata.schema?.fields ?? [];\n return fields.map((f) => f.name);\n }\n\n async getTableColumnsWithTypes(\n tableName: string,\n ): Promise<Map<string, string>> {\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const fields: Array<{ name: string; type: string }> =\n metadata.schema?.fields ?? [];\n const result = new Map<string, string>();\n for (const f of fields) {\n result.set(f.name, normalizeBigQueryType(f.type));\n }\n return result;\n }\n\n async createTable(table: SqlTableDef): Promise<void> {\n const qi = (id: string) => this.dialect.quoteIdentifier(id);\n const pk = table.columns.find((c) => c.isPrimaryKey)?.name;\n if (!pk) {\n throw new Error(\n `BigQueryAdapter requires a primary key on table \\`${table.tableName}\\` ` +\n `(Storage Write CDC mode needs PRIMARY KEY NOT ENFORCED).`,\n );\n }\n const cols = table.columns\n .map((c) => {\n const notNull = c.isPrimaryKey ? \" NOT NULL\" : \"\";\n return ` ${qi(c.name)} ${c.sqlType}${notNull}`;\n })\n .join(\",\\n\");\n\n const opts: string[] = [];\n if (this.maxStaleness !== null) {\n opts.push(`max_staleness = ${this.maxStaleness}`);\n }\n const optionsClause =\n opts.length > 0 ? `\\nOPTIONS(${opts.join(\", \")})` : \"\";\n\n const ddl =\n `CREATE TABLE IF NOT EXISTS ${this.fqn(table.tableName)} (\\n${cols},\\n` +\n ` PRIMARY KEY (${qi(pk)}) NOT ENFORCED\\n)` +\n `\\nCLUSTER BY ${qi(pk)}` +\n `${optionsClause};`;\n\n await this.bigquery.query({ query: ddl });\n }\n\n async addColumns(tableName: string, columns: SqlColumn[]): Promise<void> {\n const qi = (id: string) => this.dialect.quoteIdentifier(id);\n for (const c of columns) {\n const stmt = `ALTER TABLE ${this.fqn(tableName)} ADD COLUMN ${qi(c.name)} ${c.sqlType};`;\n await this.bigquery.query({ query: stmt });\n }\n }\n\n async executeRaw(sql: string): Promise<void> {\n await this.bigquery.query({ query: sql });\n }\n\n /**\n * Invalidate the cached writer for a table. Called by the worker after\n * `addColumns` so the next append rebuilds the proto descriptor against\n * the new schema.\n */\n onSchemaChange(tableName: string): void {\n const cached = this.writers.get(tableName);\n if (cached) {\n try {\n cached.writer.close();\n } catch {\n // ignore — connection may already be torn down\n }\n this.writers.delete(tableName);\n }\n }\n\n // -------- Inserts (Storage Write API) -----------------------------------\n\n async insertRows(\n tableName: string,\n rows: Record<string, unknown>[],\n ): Promise<void> {\n if (rows.length === 0) return;\n // Plain inserts have no PK matching, but in CDC mode every row needs a\n // _CHANGE_TYPE. We treat them as UPSERT.\n const writer = await this.getOrCreateWriter(tableName);\n const cdc = rows.map((row) => ({\n ...this.normalizeRow(row),\n _CHANGE_TYPE: \"UPSERT\",\n _CHANGE_SEQUENCE_NUMBER: formatChangeSequenceNumber(\n row[SYNC_VERSION_COLUMN],\n ),\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n async upsertRows(\n tableName: string,\n rows: Record<string, unknown>[],\n primaryKey: string,\n ): Promise<void> {\n if (rows.length === 0) return;\n const writer = await this.getOrCreateWriter(tableName, primaryKey);\n const cdc = rows.map((row) => ({\n ...this.normalizeRow(row),\n _CHANGE_TYPE: \"UPSERT\",\n _CHANGE_SEQUENCE_NUMBER: formatChangeSequenceNumber(\n row[SYNC_VERSION_COLUMN],\n ),\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n async deleteRows(\n tableName: string,\n primaryKey: string,\n ids: string[],\n ): Promise<void> {\n if (ids.length === 0) return;\n const writer = await this.getOrCreateWriter(tableName, primaryKey);\n // Storage Write CDC requires a value for every NOT-NULL column (the PK)\n // and a `_CHANGE_TYPE`. Other columns may be omitted; missing values are\n // interpreted as NULL. We pass the maximum sequence number so the DELETE\n // wins over any concurrent UPSERT for the same key — tombstones from the\n // queue carry no `__sync_version` of their own.\n const cdc = ids.map((id) => ({\n [primaryKey]: id,\n _CHANGE_TYPE: \"DELETE\",\n _CHANGE_SEQUENCE_NUMBER: \"ffffffffffffffff\",\n }));\n await this.appendWithRetry(tableName, writer, cdc);\n }\n\n // -------- Internal helpers ----------------------------------------------\n\n private get dataset() {\n return this.bigquery.dataset(this.datasetId);\n }\n\n private fqn(tableName: string): string {\n return `\\`${this.datasetId}.${tableName}\\``;\n }\n\n /** ISO 8601 timestamp pattern (e.g. 2026-03-29T20:59:27.394Z) */\n private static readonly ISO_TIMESTAMP_RE =\n /^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}/;\n\n /**\n * Convert a Date or epoch-millis number into the epoch-microseconds string\n * required by the Storage Write API for TIMESTAMP columns.\n *\n * The JSONWriter encodes through protobuf and a TIMESTAMP field is an\n * `int64`. Passing an ISO string would make `Long.fromString` throw\n * `interior hyphen` on the `-` characters.\n */\n private static toEpochMicros(d: Date): string {\n const ms = d.getTime();\n return (BigInt(ms) * 1000n).toString();\n }\n\n /** Convert JS values into shapes accepted by JSONWriter. */\n private normalizeRow(row: Record<string, unknown>): Record<string, unknown> {\n const out: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(row)) {\n if (v === undefined) continue;\n if (v instanceof Date) {\n // TIMESTAMP column → int64 epoch micros (as string to preserve precision)\n out[k] = BigQueryAdapter.toEpochMicros(v);\n } else if (\n typeof v === \"string\" &&\n BigQueryAdapter.ISO_TIMESTAMP_RE.test(v)\n ) {\n // ISO timestamp produced upstream by `serializeDocument` (Firestore\n // Timestamp → ISO string) — convert to epoch micros for protobuf.\n const d = new Date(v);\n if (!Number.isNaN(d.getTime())) {\n out[k] = BigQueryAdapter.toEpochMicros(d);\n } else {\n out[k] = v;\n }\n } else if (typeof v === \"object\" && v !== null) {\n // JSON columns: serialise nested objects/arrays\n out[k] = JSON.stringify(v);\n } else if (typeof v === \"bigint\") {\n out[k] = v.toString();\n } else {\n out[k] = v;\n }\n }\n return out;\n }\n\n private async appendWithRetry(\n tableName: string,\n writer: any,\n rows: Record<string, unknown>[],\n ): Promise<void> {\n await withRetry(async () => {\n try {\n const pending = writer.appendRows(rows);\n await pending.getResult();\n } catch (err) {\n // On certain errors (schema drift, broken connection) drop the\n // cached writer so the next call rebuilds it.\n this.onSchemaChange(tableName);\n throw err;\n }\n });\n }\n\n private async getOrCreateWriter(\n tableName: string,\n primaryKey?: string,\n ): Promise<any> {\n const cached = this.writers.get(tableName);\n if (cached) {\n if (primaryKey && cached.primaryKey !== primaryKey) {\n // PK changed — invalidate and rebuild\n this.onSchemaChange(tableName);\n } else {\n return cached.writer;\n }\n }\n\n const ns = loadStorageNs();\n if (!this.writerClient) {\n this.writerClient = new ns.managedwriter.WriterClient({\n projectId: this.projectId,\n });\n }\n\n const [metadata] = await this.dataset.table(tableName).getMetadata();\n const bqSchema = { fields: metadata.schema?.fields ?? [] };\n if (!primaryKey) {\n // Try to recover PK from table constraints if available\n const tableConstraintsPK =\n metadata.tableConstraints?.primaryKey?.columns?.[0];\n primaryKey = tableConstraintsPK ?? bqSchema.fields[0]?.name;\n }\n\n const storageSchema =\n ns.adapt.convertBigQuerySchemaToStorageTableSchema(bqSchema);\n const protoDescriptor = ns.adapt.convertStorageSchemaToProto2Descriptor(\n storageSchema,\n \"Row\",\n ns.adapt.withChangeType(),\n ns.adapt.withChangeSequenceNumber(),\n );\n\n const destinationTable = `projects/${this.projectId}/datasets/${this.datasetId}/tables/${tableName}`;\n // Pass `DefaultStream` as `streamId` (not `streamType`): the `_default`\n // stream is implicit on every table, so the client must short-circuit to\n // `${destinationTable}/streams/_default` instead of calling\n // `createWriteStream({ type: DEFAULT })` which BigQuery rejects with\n // `Unable to create a stream with type TYPE_UNSPECIFIED`.\n const connection = await this.writerClient.createStreamConnection({\n streamId: ns.managedwriter.DefaultStream,\n destinationTable,\n });\n\n const writer = new ns.managedwriter.JSONWriter({\n connection,\n protoDescriptor,\n });\n\n this.writers.set(tableName, { writer, primaryKey: primaryKey ?? \"\" });\n return writer;\n }\n\n /** Close all open writer connections. Useful for graceful shutdown. */\n async close(): Promise<void> {\n for (const { writer } of this.writers.values()) {\n try {\n writer.close();\n } catch {\n /* ignore */\n }\n }\n this.writers.clear();\n if (this.writerClient && typeof this.writerClient.close === \"function\") {\n try {\n this.writerClient.close();\n } catch {\n /* ignore */\n }\n }\n }\n}\n"]}
|