@walkeros/server-destination-gcp 0.4.2 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1 +1 @@
1
- "use strict";var e,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,o=Object.prototype.hasOwnProperty,i={};((e,n)=>{for(var r in n)t(e,r,{get:n[r],enumerable:!0})})(i,{DestinationBigQuery:()=>d,destinationBigQuery:()=>f}),module.exports=(e=i,((e,i,a,c)=>{if(i&&"object"==typeof i||"function"==typeof i)for(let s of r(i))o.call(e,s)||s===a||t(e,s,{get:()=>i[s],enumerable:!(c=n(i,s))||c.enumerable});return e})(t({},"__esModule",{value:!0}),e));var a=require("@google-cloud/bigquery");var c=Object.defineProperty;((e,t)=>{for(var n in t)c(e,n,{get:t[n],enumerable:!0})})({},{Level:()=>l});var s,l=((s=l||{})[s.ERROR=0]="ERROR",s[s.INFO=1]="INFO",s[s.DEBUG=2]="DEBUG",s);function g(e){return Array.isArray(e)}function u(e){return"object"==typeof e&&null!==e&&!g(e)&&"[object Object]"===Object.prototype.toString.call(e)}var p=e=>Object.entries(e).reduce((e,[t,n])=>(e[t]=u(n)||g(n)?JSON.stringify(n):n,e),{}),d={},f={type:"gcp-bigquery",config:{},async init({config:e,env:t,logger:n}){const r=function(e={},t,n){const r=e.settings||{},{projectId:o,bigquery:i}=r;let{client:c,location:s,datasetId:l,tableId:g}=r;o||n.throw("Config settings projectId missing"),s=s||"EU",l=l||"walkeros",g=g||"events";const u=i||{};u.projectId=o;const p=(null==t?void 0:t.BigQuery)||a.BigQuery;c=c||new p(u);const d={...r,client:c,projectId:o,location:s,datasetId:l,tableId:g};return{...e,settings:d}}(e,t,n);return r},push:async(e,{config:t,mapping:n,data:r,collector:o,env:i,logger:a})=>await async function(e,{config:t,mapping:n,data:r}){const{client:o,datasetId:i,tableId:a}=t.settings;if(!o||!i||!a)throw new Error("Missing required BigQuery configuration. Ensure init() was called successfully.");let c;if(u(r))c=r;else{const t=new Date;c={...e,timestamp:e.timestamp?new Date(e.timestamp):t,createdAt:t}}const s=[p(c)];await o.dataset(i).table(a).insert(s)}(e,{config:t,mapping:n,data:r,collector:o,env:i,logger:a})};//# sourceMappingURL=index.js.map
1
+ "use strict";var e,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.prototype.hasOwnProperty,o={};((e,n)=>{for(var r in n)t(e,r,{get:n[r],enumerable:!0})})(o,{DestinationBigQuery:()=>p,destinationBigQuery:()=>b}),module.exports=(e=o,((e,o,a,s)=>{if(o&&"object"==typeof o||"function"==typeof o)for(let c of r(o))i.call(e,c)||c===a||t(e,c,{get:()=>o[c],enumerable:!(s=n(o,c))||s.enumerable});return e})(t({},"__esModule",{value:!0}),e));var a=require("@google-cloud/bigquery");var s=Object.defineProperty;((e,t)=>{for(var n in t)s(e,n,{get:t[n],enumerable:!0})})({},{Level:()=>g});var c,g=((c=g||{})[c.ERROR=0]="ERROR",c[c.INFO=1]="INFO",c[c.DEBUG=2]="DEBUG",c);function l(e){return Array.isArray(e)}function u(e){return"object"==typeof e&&null!==e&&!l(e)&&"[object Object]"===Object.prototype.toString.call(e)}var d=e=>Object.entries(e).reduce((e,[t,n])=>(e[t]=u(n)||l(n)?JSON.stringify(n):n,e),{}),p={},b={type:"gcp-bigquery",config:{},async init({config:e,env:t,logger:n}){const r=function(e={},t,n){const r=e.settings||{},{projectId:i,bigquery:o}=r;let{client:s,location:c,datasetId:g,tableId:l}=r;i||n.throw("Config settings projectId missing"),c=c||"EU",g=g||"walkeros",l=l||"events";const u=o||{};u.projectId=i;const d=(null==t?void 0:t.BigQuery)||a.BigQuery;s=s||new d(u);const p={...r,client:s,projectId:i,location:c,datasetId:g,tableId:l};return{...e,settings:p}}(e,t,n);return r},push:async(e,{config:t,mapping:n,data:r,collector:i,env:o,logger:a})=>await async function(e,{config:t,mapping:n,data:r,logger:i}){const{client:o,datasetId:a,tableId:s}=t.settings;if(!o)return i.throw("client is missing");if(!a)return i.throw("datasetId is missing");if(!s)return i.throw("tableId is missing");let c;if(u(r))c=r;else{const t=new Date;c={...e,timestamp:e.timestamp?new Date(e.timestamp):t,createdAt:t}}const g=[d(c)];i.debug("Calling BigQuery API",{dataset:a,table:s,rowCount:g.length}),await o.dataset(a).table(s).insert(g),i.debug("BigQuery API response",{ok:!0})}(e,{config:t,mapping:n,data:r,collector:i,env:o,logger:a})};//# sourceMappingURL=index.js.map
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/index.ts","../src/bigquery/config.ts","../src/bigquery/push.ts","../src/bigquery/types/index.ts","../src/bigquery/index.ts"],"sourcesContent":["// Google BigQuery\nexport { destinationBigQuery } from './bigquery';\nexport * as DestinationBigQuery from './bigquery/types';\n","import type { Config, Env, PartialConfig, Settings } from './types';\nimport type { BigQueryOptions } from '@google-cloud/bigquery';\nimport type { Logger } from '@walkeros/core';\nimport { BigQuery } from '@google-cloud/bigquery';\n\nexport function getConfig(\n partialConfig: PartialConfig = {},\n env: Env | undefined,\n logger: Logger.Instance,\n): Config {\n const settings = partialConfig.settings || ({} as Settings);\n const { projectId, bigquery } = settings;\n let { client, location, datasetId, tableId } = settings;\n\n if (!projectId) logger.throw('Config settings projectId missing');\n\n location = location || 'EU';\n datasetId = datasetId || 'walkeros';\n tableId = tableId || 'events';\n\n const options: BigQueryOptions = bigquery || {};\n options.projectId = projectId;\n\n // Use BigQuery from env if available, otherwise use real BigQuery\n const BigQueryClass = env?.BigQuery || BigQuery;\n client = client || new BigQueryClass(options);\n\n const settingsConfig: Settings = {\n ...settings,\n client,\n projectId,\n location,\n datasetId,\n tableId,\n };\n\n return { ...partialConfig, settings: settingsConfig };\n}\n","import type { WalkerOS } from '@walkeros/core';\nimport type { PushFn } from './types';\nimport { isObject, isArray } from '@walkeros/core';\n\nexport const push: PushFn = async function (\n event,\n { config, mapping: _mapping, data },\n) {\n const { client, datasetId, tableId } = config.settings!;\n\n if (!client || !datasetId || !tableId) {\n throw new Error(\n 'Missing required BigQuery configuration. Ensure init() was called successfully.',\n );\n }\n\n let row: WalkerOS.AnyObject | undefined;\n\n if (isObject(data)) {\n row = data;\n } else {\n const now = new Date();\n row = {\n ...event,\n timestamp: event.timestamp ? new Date(event.timestamp) : now,\n createdAt: now,\n };\n }\n\n const rows = [mapEvent(row)];\n\n await client.dataset(datasetId).table(tableId).insert(rows);\n\n return;\n};\n\nexport const mapEvent = (event: WalkerOS.AnyObject) => {\n return Object.entries(event).reduce<WalkerOS.AnyObject>(\n (acc, [key, value]) => {\n acc[key] =\n isObject(value) || isArray(value) ? JSON.stringify(value) : value;\n return acc;\n },\n {},\n );\n};\n","import type { DestinationServer } from '@walkeros/server-core';\nimport type {\n Mapping as WalkerOSMapping,\n Destination as CoreDestination,\n} from '@walkeros/core';\nimport type { BigQuery, BigQueryOptions } from '@google-cloud/bigquery';\n\nexport interface Settings {\n client: BigQuery;\n projectId: string;\n datasetId: string;\n tableId: string;\n location?: string;\n bigquery?: BigQueryOptions;\n}\n\nexport interface InitSettings {\n projectId: string;\n client?: BigQuery;\n datasetId?: string;\n tableId?: string;\n location?: string;\n bigquery?: BigQueryOptions;\n}\n\nexport interface Mapping {}\n\nexport interface Env extends DestinationServer.Env {\n BigQuery?: typeof BigQuery;\n}\n\nexport type Types = CoreDestination.Types<Settings, Mapping, Env, InitSettings>;\n\nexport interface Destination extends DestinationServer.Destination<Types> {\n init: DestinationServer.InitFn<Types>;\n}\n\nexport type Config = {\n settings: Settings;\n} & DestinationServer.Config<Types>;\n\nexport type InitFn = DestinationServer.InitFn<Types>;\nexport type PushFn = DestinationServer.PushFn<Types>;\n\nexport type PartialConfig = DestinationServer.PartialConfig<Types>;\n\nexport type PushEvents = DestinationServer.PushEvents<Mapping>;\n\nexport type Rule = WalkerOSMapping.Rule<Mapping>;\nexport type Rules = WalkerOSMapping.Rules<Rule>;\n","import type { Destination } from './types';\nimport { getConfig } from './config';\nimport { push } from './push';\n\n// Types\nexport * as DestinationBigQuery from './types';\n\nexport const destinationBigQuery: Destination = {\n type: 'gcp-bigquery',\n\n config: {},\n\n async init({ config: partialConfig, env, logger }) {\n const config = getConfig(partialConfig, env, logger);\n\n return config;\n },\n\n async push(event, { config, mapping, data, collector, env, logger }) {\n return await push(event, { config, mapping, data, collector, env, logger });\n },\n};\n\nexport default destinationBigQuery;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACGA,sBAAyB;AAElB,SAAS,UACd,gBAA+B,CAAC,GAChC,KACA,QACQ;AACR,QAAM,WAAW,cAAc,YAAa,CAAC;AAC7C,QAAM,EAAE,WAAW,SAAS,IAAI;AAChC,MAAI,EAAE,QAAQ,UAAU,WAAW,QAAQ,IAAI;AAE/C,MAAI,CAAC,UAAW,QAAO,MAAM,mCAAmC;AAEhE,aAAW,YAAY;AACvB,cAAY,aAAa;AACzB,YAAU,WAAW;AAErB,QAAM,UAA2B,YAAY,CAAC;AAC9C,UAAQ,YAAY;AAGpB,QAAM,iBAAgB,2BAAK,aAAY;AACvC,WAAS,UAAU,IAAI,cAAc,OAAO;AAE5C,QAAM,iBAA2B;AAAA,IAC/B,GAAG;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,SAAO,EAAE,GAAG,eAAe,UAAU,eAAe;AACtD;A;;;;;;;;;;;;;;;;ACjCO,IAAM,OAAe,eAC1B,OACA,EAAE,QAAQ,SAAS,UAAU,KAAK,GAClC;AACA,QAAM,EAAE,QAAQ,WAAW,QAAQ,IAAI,OAAO;AAE9C,MAAI,CAAC,UAAU,CAAC,aAAa,CAAC,SAAS;AACrC,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,MAAI;AAEJ,MAAI,EAAS,IAAI,GAAG;AAClB,UAAM;AAAA,EACR,OAAO;AACL,UAAM,MAAM,oBAAI,KAAK;AACrB,UAAM;AAAA,MACJ,GAAG;AAAA,MACH,WAAW,MAAM,YAAY,IAAI,KAAK,MAAM,SAAS,IAAI;AAAA,MACzD,WAAW;AAAA,IACb;AAAA,EACF;AAEA,QAAM,OAAO,CAAC,SAAS,GAAG,CAAC;AAE3B,QAAM,OAAO,QAAQ,SAAS,EAAE,MAAM,OAAO,EAAE,OAAO,IAAI;AAE1D;AACF;AAEO,IAAM,WAAW,CAAC,UAA8B;AACrD,SAAO,OAAO,QAAQ,KAAK,EAAE;AAAA,IAC3B,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM;AACrB,UAAI,GAAG,IACL,EAAS,KAAK,KAAK,EAAQ,KAAK,IAAI,KAAK,UAAU,KAAK,IAAI;AAC9D,aAAO;AAAA,IACT;AAAA,IACA,CAAC;AAAA,EACH;AACF;;;AC7CA;;;ACOO,IAAM,sBAAmC;AAAA,EAC9C,MAAM;AAAA,EAEN,QAAQ,CAAC;AAAA,EAET,MAAM,KAAK,EAAE,QAAQ,eAAe,KAAK,OAAO,GAAG;AACjD,UAAM,SAAS,UAAU,eAAe,KAAK,MAAM;AAEnD,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,KAAK,OAAO,EAAE,QAAQ,SAAS,MAAM,WAAW,KAAK,OAAO,GAAG;AACnE,WAAO,MAAM,KAAK,OAAO,EAAE,QAAQ,SAAS,MAAM,WAAW,KAAK,OAAO,CAAC;AAAA,EAC5E;AACF;","names":[]}
1
+ {"version":3,"sources":["../src/index.ts","../src/bigquery/config.ts","../src/bigquery/push.ts","../src/bigquery/types/index.ts","../src/bigquery/index.ts"],"sourcesContent":["// Google BigQuery\nexport { destinationBigQuery } from './bigquery';\nexport * as DestinationBigQuery from './bigquery/types';\n","import type { Config, Env, PartialConfig, Settings } from './types';\nimport type { BigQueryOptions } from '@google-cloud/bigquery';\nimport type { Logger } from '@walkeros/core';\nimport { BigQuery } from '@google-cloud/bigquery';\n\nexport function getConfig(\n partialConfig: PartialConfig = {},\n env: Env | undefined,\n logger: Logger.Instance,\n): Config {\n const settings = partialConfig.settings || ({} as Settings);\n const { projectId, bigquery } = settings;\n let { client, location, datasetId, tableId } = settings;\n\n if (!projectId) logger.throw('Config settings projectId missing');\n\n location = location || 'EU';\n datasetId = datasetId || 'walkeros';\n tableId = tableId || 'events';\n\n const options: BigQueryOptions = bigquery || {};\n options.projectId = projectId;\n\n // Use BigQuery from env if available, otherwise use real BigQuery\n const BigQueryClass = env?.BigQuery || BigQuery;\n client = client || new BigQueryClass(options);\n\n const settingsConfig: Settings = {\n ...settings,\n client,\n projectId,\n location,\n datasetId,\n tableId,\n };\n\n return { ...partialConfig, settings: settingsConfig };\n}\n","import type { WalkerOS } from '@walkeros/core';\nimport type { PushFn } from './types';\nimport { isObject, isArray } from '@walkeros/core';\n\nexport const push: PushFn = async function (\n event,\n { config, mapping: _mapping, data, logger },\n) {\n const { client, datasetId, tableId } = config.settings!;\n\n if (!client) return logger.throw('client is missing');\n if (!datasetId) return logger.throw('datasetId is missing');\n if (!tableId) return logger.throw('tableId is missing');\n\n let row: WalkerOS.AnyObject | undefined;\n\n if (isObject(data)) {\n row = data;\n } else {\n const now = new Date();\n row = {\n ...event,\n timestamp: event.timestamp ? new Date(event.timestamp) : now,\n createdAt: now,\n };\n }\n\n const rows = [mapEvent(row)];\n\n logger.debug('Calling BigQuery API', {\n dataset: datasetId,\n table: tableId,\n rowCount: rows.length,\n });\n\n await client.dataset(datasetId).table(tableId).insert(rows);\n\n logger.debug('BigQuery API response', { ok: true });\n\n return;\n};\n\nexport const mapEvent = (event: WalkerOS.AnyObject) => {\n return Object.entries(event).reduce<WalkerOS.AnyObject>(\n (acc, [key, value]) => {\n acc[key] =\n isObject(value) || isArray(value) ? JSON.stringify(value) : value;\n return acc;\n },\n {},\n );\n};\n","import type { DestinationServer } from '@walkeros/server-core';\nimport type {\n Mapping as WalkerOSMapping,\n Destination as CoreDestination,\n} from '@walkeros/core';\nimport type { BigQuery, BigQueryOptions } from '@google-cloud/bigquery';\n\nexport interface Settings {\n client: BigQuery;\n projectId: string;\n datasetId: string;\n tableId: string;\n location?: string;\n bigquery?: BigQueryOptions;\n}\n\nexport interface InitSettings {\n projectId: string;\n client?: BigQuery;\n datasetId?: string;\n tableId?: string;\n location?: string;\n bigquery?: BigQueryOptions;\n}\n\nexport interface Mapping {}\n\nexport interface Env extends DestinationServer.Env {\n BigQuery?: typeof BigQuery;\n}\n\nexport type Types = CoreDestination.Types<Settings, Mapping, Env, InitSettings>;\n\nexport interface Destination extends DestinationServer.Destination<Types> {\n init: DestinationServer.InitFn<Types>;\n}\n\nexport type Config = {\n settings: Settings;\n} & DestinationServer.Config<Types>;\n\nexport type InitFn = DestinationServer.InitFn<Types>;\nexport type PushFn = DestinationServer.PushFn<Types>;\n\nexport type PartialConfig = DestinationServer.PartialConfig<Types>;\n\nexport type PushEvents = DestinationServer.PushEvents<Mapping>;\n\nexport type Rule = WalkerOSMapping.Rule<Mapping>;\nexport type Rules = WalkerOSMapping.Rules<Rule>;\n","import type { Destination } from './types';\nimport { getConfig } from './config';\nimport { push } from './push';\n\n// Types\nexport * as DestinationBigQuery from './types';\n\nexport const destinationBigQuery: Destination = {\n type: 'gcp-bigquery',\n\n config: {},\n\n async init({ config: partialConfig, env, logger }) {\n const config = getConfig(partialConfig, env, logger);\n\n return config;\n },\n\n async push(event, { config, mapping, data, collector, env, logger }) {\n return await push(event, { config, mapping, data, collector, env, logger });\n },\n};\n\nexport default destinationBigQuery;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACGA,sBAAyB;AAElB,SAAS,UACd,gBAA+B,CAAC,GAChC,KACA,QACQ;AACR,QAAM,WAAW,cAAc,YAAa,CAAC;AAC7C,QAAM,EAAE,WAAW,SAAS,IAAI;AAChC,MAAI,EAAE,QAAQ,UAAU,WAAW,QAAQ,IAAI;AAE/C,MAAI,CAAC,UAAW,QAAO,MAAM,mCAAmC;AAEhE,aAAW,YAAY;AACvB,cAAY,aAAa;AACzB,YAAU,WAAW;AAErB,QAAM,UAA2B,YAAY,CAAC;AAC9C,UAAQ,YAAY;AAGpB,QAAM,iBAAgB,2BAAK,aAAY;AACvC,WAAS,UAAU,IAAI,cAAc,OAAO;AAE5C,QAAM,iBAA2B;AAAA,IAC/B,GAAG;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,SAAO,EAAE,GAAG,eAAe,UAAU,eAAe;AACtD;A;;;;;;;;;;;;;;;;ACjCO,IAAM,OAAe,eAC1B,OACA,EAAE,QAAQ,SAAS,UAAU,MAAM,OAAO,GAC1C;AACA,QAAM,EAAE,QAAQ,WAAW,QAAQ,IAAI,OAAO;AAE9C,MAAI,CAAC,OAAQ,QAAO,OAAO,MAAM,mBAAmB;AACpD,MAAI,CAAC,UAAW,QAAO,OAAO,MAAM,sBAAsB;AAC1D,MAAI,CAAC,QAAS,QAAO,OAAO,MAAM,oBAAoB;AAEtD,MAAI;AAEJ,MAAI,EAAS,IAAI,GAAG;AAClB,UAAM;AAAA,EACR,OAAO;AACL,UAAM,MAAM,oBAAI,KAAK;AACrB,UAAM;AAAA,MACJ,GAAG;AAAA,MACH,WAAW,MAAM,YAAY,IAAI,KAAK,MAAM,SAAS,IAAI;AAAA,MACzD,WAAW;AAAA,IACb;AAAA,EACF;AAEA,QAAM,OAAO,CAAC,SAAS,GAAG,CAAC;AAE3B,SAAO,MAAM,wBAAwB;AAAA,IACnC,SAAS;AAAA,IACT,OAAO;AAAA,IACP,UAAU,KAAK;AAAA,EACjB,CAAC;AAED,QAAM,OAAO,QAAQ,SAAS,EAAE,MAAM,OAAO,EAAE,OAAO,IAAI;AAE1D,SAAO,MAAM,yBAAyB,EAAE,IAAI,KAAK,CAAC;AAElD;AACF;AAEO,IAAM,WAAW,CAAC,UAA8B;AACrD,SAAO,OAAO,QAAQ,KAAK,EAAE;AAAA,IAC3B,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM;AACrB,UAAI,GAAG,IACL,EAAS,KAAK,KAAK,EAAQ,KAAK,IAAI,KAAK,UAAU,KAAK,IAAI;AAC9D,aAAO;AAAA,IACT;AAAA,IACA,CAAC;AAAA,EACH;AACF;;;ACnDA;;;ACOO,IAAM,sBAAmC;AAAA,EAC9C,MAAM;AAAA,EAEN,QAAQ,CAAC;AAAA,EAET,MAAM,KAAK,EAAE,QAAQ,eAAe,KAAK,OAAO,GAAG;AACjD,UAAM,SAAS,UAAU,eAAe,KAAK,MAAM;AAEnD,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,KAAK,OAAO,EAAE,QAAQ,SAAS,MAAM,WAAW,KAAK,OAAO,GAAG;AACnE,WAAO,MAAM,KAAK,OAAO,EAAE,QAAQ,SAAS,MAAM,WAAW,KAAK,OAAO,CAAC;AAAA,EAC5E;AACF;","names":[]}
package/dist/index.mjs CHANGED
@@ -1 +1 @@
1
- import{BigQuery as t}from"@google-cloud/bigquery";var e=Object.defineProperty;((t,n)=>{for(var i in n)e(t,i,{get:n[i],enumerable:!0})})({},{Level:()=>i});var n,i=((n=i||{})[n.ERROR=0]="ERROR",n[n.INFO=1]="INFO",n[n.DEBUG=2]="DEBUG",n);function o(t){return Array.isArray(t)}function r(t){return"object"==typeof t&&null!==t&&!o(t)&&"[object Object]"===Object.prototype.toString.call(t)}var a=t=>Object.entries(t).reduce((t,[e,n])=>(t[e]=r(n)||o(n)?JSON.stringify(n):n,t),{}),c={},s={type:"gcp-bigquery",config:{},async init({config:e,env:n,logger:i}){const o=function(e={},n,i){const o=e.settings||{},{projectId:r,bigquery:a}=o;let{client:c,location:s,datasetId:g,tableId:l}=o;r||i.throw("Config settings projectId missing"),s=s||"EU",g=g||"walkeros",l=l||"events";const u=a||{};u.projectId=r;const d=(null==n?void 0:n.BigQuery)||t;c=c||new d(u);const p={...o,client:c,projectId:r,location:s,datasetId:g,tableId:l};return{...e,settings:p}}(e,n,i);return o},push:async(t,{config:e,mapping:n,data:i,collector:o,env:c,logger:s})=>await async function(t,{config:e,mapping:n,data:i}){const{client:o,datasetId:c,tableId:s}=e.settings;if(!o||!c||!s)throw new Error("Missing required BigQuery configuration. Ensure init() was called successfully.");let g;if(r(i))g=i;else{const e=new Date;g={...t,timestamp:t.timestamp?new Date(t.timestamp):e,createdAt:e}}const l=[a(g)];await o.dataset(c).table(s).insert(l)}(t,{config:e,mapping:n,data:i,collector:o,env:c,logger:s})};export{c as DestinationBigQuery,s as destinationBigQuery};//# sourceMappingURL=index.mjs.map
1
+ import{BigQuery as t}from"@google-cloud/bigquery";var e=Object.defineProperty;((t,n)=>{for(var i in n)e(t,i,{get:n[i],enumerable:!0})})({},{Level:()=>i});var n,i=((n=i||{})[n.ERROR=0]="ERROR",n[n.INFO=1]="INFO",n[n.DEBUG=2]="DEBUG",n);function r(t){return Array.isArray(t)}function o(t){return"object"==typeof t&&null!==t&&!r(t)&&"[object Object]"===Object.prototype.toString.call(t)}var a=t=>Object.entries(t).reduce((t,[e,n])=>(t[e]=o(n)||r(n)?JSON.stringify(n):n,t),{}),s={},c={type:"gcp-bigquery",config:{},async init({config:e,env:n,logger:i}){const r=function(e={},n,i){const r=e.settings||{},{projectId:o,bigquery:a}=r;let{client:s,location:c,datasetId:g,tableId:l}=r;o||i.throw("Config settings projectId missing"),c=c||"EU",g=g||"walkeros",l=l||"events";const d=a||{};d.projectId=o;const u=(null==n?void 0:n.BigQuery)||t;s=s||new u(d);const p={...r,client:s,projectId:o,location:c,datasetId:g,tableId:l};return{...e,settings:p}}(e,n,i);return r},push:async(t,{config:e,mapping:n,data:i,collector:r,env:s,logger:c})=>await async function(t,{config:e,mapping:n,data:i,logger:r}){const{client:s,datasetId:c,tableId:g}=e.settings;if(!s)return r.throw("client is missing");if(!c)return r.throw("datasetId is missing");if(!g)return r.throw("tableId is missing");let l;if(o(i))l=i;else{const e=new Date;l={...t,timestamp:t.timestamp?new Date(t.timestamp):e,createdAt:e}}const d=[a(l)];r.debug("Calling BigQuery API",{dataset:c,table:g,rowCount:d.length}),await s.dataset(c).table(g).insert(d),r.debug("BigQuery API response",{ok:!0})}(t,{config:e,mapping:n,data:i,collector:r,env:s,logger:c})};export{s as DestinationBigQuery,c as destinationBigQuery};//# sourceMappingURL=index.mjs.map
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/bigquery/config.ts","../src/bigquery/push.ts","../src/bigquery/types/index.ts","../src/bigquery/index.ts"],"sourcesContent":["import type { Config, Env, PartialConfig, Settings } from './types';\nimport type { BigQueryOptions } from '@google-cloud/bigquery';\nimport type { Logger } from '@walkeros/core';\nimport { BigQuery } from '@google-cloud/bigquery';\n\nexport function getConfig(\n partialConfig: PartialConfig = {},\n env: Env | undefined,\n logger: Logger.Instance,\n): Config {\n const settings = partialConfig.settings || ({} as Settings);\n const { projectId, bigquery } = settings;\n let { client, location, datasetId, tableId } = settings;\n\n if (!projectId) logger.throw('Config settings projectId missing');\n\n location = location || 'EU';\n datasetId = datasetId || 'walkeros';\n tableId = tableId || 'events';\n\n const options: BigQueryOptions = bigquery || {};\n options.projectId = projectId;\n\n // Use BigQuery from env if available, otherwise use real BigQuery\n const BigQueryClass = env?.BigQuery || BigQuery;\n client = client || new BigQueryClass(options);\n\n const settingsConfig: Settings = {\n ...settings,\n client,\n projectId,\n location,\n datasetId,\n tableId,\n };\n\n return { ...partialConfig, settings: settingsConfig };\n}\n","import type { WalkerOS } from '@walkeros/core';\nimport type { PushFn } from './types';\nimport { isObject, isArray } from '@walkeros/core';\n\nexport const push: PushFn = async function (\n event,\n { config, mapping: _mapping, data },\n) {\n const { client, datasetId, tableId } = config.settings!;\n\n if (!client || !datasetId || !tableId) {\n throw new Error(\n 'Missing required BigQuery configuration. Ensure init() was called successfully.',\n );\n }\n\n let row: WalkerOS.AnyObject | undefined;\n\n if (isObject(data)) {\n row = data;\n } else {\n const now = new Date();\n row = {\n ...event,\n timestamp: event.timestamp ? new Date(event.timestamp) : now,\n createdAt: now,\n };\n }\n\n const rows = [mapEvent(row)];\n\n await client.dataset(datasetId).table(tableId).insert(rows);\n\n return;\n};\n\nexport const mapEvent = (event: WalkerOS.AnyObject) => {\n return Object.entries(event).reduce<WalkerOS.AnyObject>(\n (acc, [key, value]) => {\n acc[key] =\n isObject(value) || isArray(value) ? JSON.stringify(value) : value;\n return acc;\n },\n {},\n );\n};\n","import type { DestinationServer } from '@walkeros/server-core';\nimport type {\n Mapping as WalkerOSMapping,\n Destination as CoreDestination,\n} from '@walkeros/core';\nimport type { BigQuery, BigQueryOptions } from '@google-cloud/bigquery';\n\nexport interface Settings {\n client: BigQuery;\n projectId: string;\n datasetId: string;\n tableId: string;\n location?: string;\n bigquery?: BigQueryOptions;\n}\n\nexport interface InitSettings {\n projectId: string;\n client?: BigQuery;\n datasetId?: string;\n tableId?: string;\n location?: string;\n bigquery?: BigQueryOptions;\n}\n\nexport interface Mapping {}\n\nexport interface Env extends DestinationServer.Env {\n BigQuery?: typeof BigQuery;\n}\n\nexport type Types = CoreDestination.Types<Settings, Mapping, Env, InitSettings>;\n\nexport interface Destination extends DestinationServer.Destination<Types> {\n init: DestinationServer.InitFn<Types>;\n}\n\nexport type Config = {\n settings: Settings;\n} & DestinationServer.Config<Types>;\n\nexport type InitFn = DestinationServer.InitFn<Types>;\nexport type PushFn = DestinationServer.PushFn<Types>;\n\nexport type PartialConfig = DestinationServer.PartialConfig<Types>;\n\nexport type PushEvents = DestinationServer.PushEvents<Mapping>;\n\nexport type Rule = WalkerOSMapping.Rule<Mapping>;\nexport type Rules = WalkerOSMapping.Rules<Rule>;\n","import type { Destination } from './types';\nimport { getConfig } from './config';\nimport { push } from './push';\n\n// Types\nexport * as DestinationBigQuery from './types';\n\nexport const destinationBigQuery: Destination = {\n type: 'gcp-bigquery',\n\n config: {},\n\n async init({ config: partialConfig, env, logger }) {\n const config = getConfig(partialConfig, env, logger);\n\n return config;\n },\n\n async push(event, { config, mapping, data, collector, env, logger }) {\n return await push(event, { config, mapping, data, collector, env, logger });\n },\n};\n\nexport default destinationBigQuery;\n"],"mappings":";AAGA,SAAS,gBAAgB;AAElB,SAAS,UACd,gBAA+B,CAAC,GAChC,KACA,QACQ;AACR,QAAM,WAAW,cAAc,YAAa,CAAC;AAC7C,QAAM,EAAE,WAAW,SAAS,IAAI;AAChC,MAAI,EAAE,QAAQ,UAAU,WAAW,QAAQ,IAAI;AAE/C,MAAI,CAAC,UAAW,QAAO,MAAM,mCAAmC;AAEhE,aAAW,YAAY;AACvB,cAAY,aAAa;AACzB,YAAU,WAAW;AAErB,QAAM,UAA2B,YAAY,CAAC;AAC9C,UAAQ,YAAY;AAGpB,QAAM,iBAAgB,2BAAK,aAAY;AACvC,WAAS,UAAU,IAAI,cAAc,OAAO;AAE5C,QAAM,iBAA2B;AAAA,IAC/B,GAAG;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,SAAO,EAAE,GAAG,eAAe,UAAU,eAAe;AACtD;A;;;;;;;;;;;;;;;;ACjCO,IAAM,OAAe,eAC1B,OACA,EAAE,QAAQ,SAAS,UAAU,KAAK,GAClC;AACA,QAAM,EAAE,QAAQ,WAAW,QAAQ,IAAI,OAAO;AAE9C,MAAI,CAAC,UAAU,CAAC,aAAa,CAAC,SAAS;AACrC,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,MAAI;AAEJ,MAAI,EAAS,IAAI,GAAG;AAClB,UAAM;AAAA,EACR,OAAO;AACL,UAAM,MAAM,oBAAI,KAAK;AACrB,UAAM;AAAA,MACJ,GAAG;AAAA,MACH,WAAW,MAAM,YAAY,IAAI,KAAK,MAAM,SAAS,IAAI;AAAA,MACzD,WAAW;AAAA,IACb;AAAA,EACF;AAEA,QAAM,OAAO,CAAC,SAAS,GAAG,CAAC;AAE3B,QAAM,OAAO,QAAQ,SAAS,EAAE,MAAM,OAAO,EAAE,OAAO,IAAI;AAE1D;AACF;AAEO,IAAM,WAAW,CAAC,UAA8B;AACrD,SAAO,OAAO,QAAQ,KAAK,EAAE;AAAA,IAC3B,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM;AACrB,UAAI,GAAG,IACL,EAAS,KAAK,KAAK,EAAQ,KAAK,IAAI,KAAK,UAAU,KAAK,IAAI;AAC9D,aAAO;AAAA,IACT;AAAA,IACA,CAAC;AAAA,EACH;AACF;;;AC7CA;;;ACOO,IAAM,sBAAmC;AAAA,EAC9C,MAAM;AAAA,EAEN,QAAQ,CAAC;AAAA,EAET,MAAM,KAAK,EAAE,QAAQ,eAAe,KAAK,OAAO,GAAG;AACjD,UAAM,SAAS,UAAU,eAAe,KAAK,MAAM;AAEnD,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,KAAK,OAAO,EAAE,QAAQ,SAAS,MAAM,WAAW,KAAK,OAAO,GAAG;AACnE,WAAO,MAAM,KAAK,OAAO,EAAE,QAAQ,SAAS,MAAM,WAAW,KAAK,OAAO,CAAC;AAAA,EAC5E;AACF;","names":[]}
1
+ {"version":3,"sources":["../src/bigquery/config.ts","../src/bigquery/push.ts","../src/bigquery/types/index.ts","../src/bigquery/index.ts"],"sourcesContent":["import type { Config, Env, PartialConfig, Settings } from './types';\nimport type { BigQueryOptions } from '@google-cloud/bigquery';\nimport type { Logger } from '@walkeros/core';\nimport { BigQuery } from '@google-cloud/bigquery';\n\nexport function getConfig(\n partialConfig: PartialConfig = {},\n env: Env | undefined,\n logger: Logger.Instance,\n): Config {\n const settings = partialConfig.settings || ({} as Settings);\n const { projectId, bigquery } = settings;\n let { client, location, datasetId, tableId } = settings;\n\n if (!projectId) logger.throw('Config settings projectId missing');\n\n location = location || 'EU';\n datasetId = datasetId || 'walkeros';\n tableId = tableId || 'events';\n\n const options: BigQueryOptions = bigquery || {};\n options.projectId = projectId;\n\n // Use BigQuery from env if available, otherwise use real BigQuery\n const BigQueryClass = env?.BigQuery || BigQuery;\n client = client || new BigQueryClass(options);\n\n const settingsConfig: Settings = {\n ...settings,\n client,\n projectId,\n location,\n datasetId,\n tableId,\n };\n\n return { ...partialConfig, settings: settingsConfig };\n}\n","import type { WalkerOS } from '@walkeros/core';\nimport type { PushFn } from './types';\nimport { isObject, isArray } from '@walkeros/core';\n\nexport const push: PushFn = async function (\n event,\n { config, mapping: _mapping, data, logger },\n) {\n const { client, datasetId, tableId } = config.settings!;\n\n if (!client) return logger.throw('client is missing');\n if (!datasetId) return logger.throw('datasetId is missing');\n if (!tableId) return logger.throw('tableId is missing');\n\n let row: WalkerOS.AnyObject | undefined;\n\n if (isObject(data)) {\n row = data;\n } else {\n const now = new Date();\n row = {\n ...event,\n timestamp: event.timestamp ? new Date(event.timestamp) : now,\n createdAt: now,\n };\n }\n\n const rows = [mapEvent(row)];\n\n logger.debug('Calling BigQuery API', {\n dataset: datasetId,\n table: tableId,\n rowCount: rows.length,\n });\n\n await client.dataset(datasetId).table(tableId).insert(rows);\n\n logger.debug('BigQuery API response', { ok: true });\n\n return;\n};\n\nexport const mapEvent = (event: WalkerOS.AnyObject) => {\n return Object.entries(event).reduce<WalkerOS.AnyObject>(\n (acc, [key, value]) => {\n acc[key] =\n isObject(value) || isArray(value) ? JSON.stringify(value) : value;\n return acc;\n },\n {},\n );\n};\n","import type { DestinationServer } from '@walkeros/server-core';\nimport type {\n Mapping as WalkerOSMapping,\n Destination as CoreDestination,\n} from '@walkeros/core';\nimport type { BigQuery, BigQueryOptions } from '@google-cloud/bigquery';\n\nexport interface Settings {\n client: BigQuery;\n projectId: string;\n datasetId: string;\n tableId: string;\n location?: string;\n bigquery?: BigQueryOptions;\n}\n\nexport interface InitSettings {\n projectId: string;\n client?: BigQuery;\n datasetId?: string;\n tableId?: string;\n location?: string;\n bigquery?: BigQueryOptions;\n}\n\nexport interface Mapping {}\n\nexport interface Env extends DestinationServer.Env {\n BigQuery?: typeof BigQuery;\n}\n\nexport type Types = CoreDestination.Types<Settings, Mapping, Env, InitSettings>;\n\nexport interface Destination extends DestinationServer.Destination<Types> {\n init: DestinationServer.InitFn<Types>;\n}\n\nexport type Config = {\n settings: Settings;\n} & DestinationServer.Config<Types>;\n\nexport type InitFn = DestinationServer.InitFn<Types>;\nexport type PushFn = DestinationServer.PushFn<Types>;\n\nexport type PartialConfig = DestinationServer.PartialConfig<Types>;\n\nexport type PushEvents = DestinationServer.PushEvents<Mapping>;\n\nexport type Rule = WalkerOSMapping.Rule<Mapping>;\nexport type Rules = WalkerOSMapping.Rules<Rule>;\n","import type { Destination } from './types';\nimport { getConfig } from './config';\nimport { push } from './push';\n\n// Types\nexport * as DestinationBigQuery from './types';\n\nexport const destinationBigQuery: Destination = {\n type: 'gcp-bigquery',\n\n config: {},\n\n async init({ config: partialConfig, env, logger }) {\n const config = getConfig(partialConfig, env, logger);\n\n return config;\n },\n\n async push(event, { config, mapping, data, collector, env, logger }) {\n return await push(event, { config, mapping, data, collector, env, logger });\n },\n};\n\nexport default destinationBigQuery;\n"],"mappings":";AAGA,SAAS,gBAAgB;AAElB,SAAS,UACd,gBAA+B,CAAC,GAChC,KACA,QACQ;AACR,QAAM,WAAW,cAAc,YAAa,CAAC;AAC7C,QAAM,EAAE,WAAW,SAAS,IAAI;AAChC,MAAI,EAAE,QAAQ,UAAU,WAAW,QAAQ,IAAI;AAE/C,MAAI,CAAC,UAAW,QAAO,MAAM,mCAAmC;AAEhE,aAAW,YAAY;AACvB,cAAY,aAAa;AACzB,YAAU,WAAW;AAErB,QAAM,UAA2B,YAAY,CAAC;AAC9C,UAAQ,YAAY;AAGpB,QAAM,iBAAgB,2BAAK,aAAY;AACvC,WAAS,UAAU,IAAI,cAAc,OAAO;AAE5C,QAAM,iBAA2B;AAAA,IAC/B,GAAG;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,SAAO,EAAE,GAAG,eAAe,UAAU,eAAe;AACtD;A;;;;;;;;;;;;;;;;ACjCO,IAAM,OAAe,eAC1B,OACA,EAAE,QAAQ,SAAS,UAAU,MAAM,OAAO,GAC1C;AACA,QAAM,EAAE,QAAQ,WAAW,QAAQ,IAAI,OAAO;AAE9C,MAAI,CAAC,OAAQ,QAAO,OAAO,MAAM,mBAAmB;AACpD,MAAI,CAAC,UAAW,QAAO,OAAO,MAAM,sBAAsB;AAC1D,MAAI,CAAC,QAAS,QAAO,OAAO,MAAM,oBAAoB;AAEtD,MAAI;AAEJ,MAAI,EAAS,IAAI,GAAG;AAClB,UAAM;AAAA,EACR,OAAO;AACL,UAAM,MAAM,oBAAI,KAAK;AACrB,UAAM;AAAA,MACJ,GAAG;AAAA,MACH,WAAW,MAAM,YAAY,IAAI,KAAK,MAAM,SAAS,IAAI;AAAA,MACzD,WAAW;AAAA,IACb;AAAA,EACF;AAEA,QAAM,OAAO,CAAC,SAAS,GAAG,CAAC;AAE3B,SAAO,MAAM,wBAAwB;AAAA,IACnC,SAAS;AAAA,IACT,OAAO;AAAA,IACP,UAAU,KAAK;AAAA,EACjB,CAAC;AAED,QAAM,OAAO,QAAQ,SAAS,EAAE,MAAM,OAAO,EAAE,OAAO,IAAI;AAE1D,SAAO,MAAM,yBAAyB,EAAE,IAAI,KAAK,CAAC;AAElD;AACF;AAEO,IAAM,WAAW,CAAC,UAA8B;AACrD,SAAO,OAAO,QAAQ,KAAK,EAAE;AAAA,IAC3B,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM;AACrB,UAAI,GAAG,IACL,EAAS,KAAK,KAAK,EAAQ,KAAK,IAAI,KAAK,UAAU,KAAK,IAAI;AAC9D,aAAO;AAAA,IACT;AAAA,IACA,CAAC;AAAA,EACH;AACF;;;ACnDA;;;ACOO,IAAM,sBAAmC;AAAA,EAC9C,MAAM;AAAA,EAEN,QAAQ,CAAC;AAAA,EAET,MAAM,KAAK,EAAE,QAAQ,eAAe,KAAK,OAAO,GAAG;AACjD,UAAM,SAAS,UAAU,eAAe,KAAK,MAAM;AAEnD,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,KAAK,OAAO,EAAE,QAAQ,SAAS,MAAM,WAAW,KAAK,OAAO,GAAG;AACnE,WAAO,MAAM,KAAK,OAAO,EAAE,QAAQ,SAAS,MAAM,WAAW,KAAK,OAAO,CAAC;AAAA,EAC5E;AACF;","names":[]}