@walkeros/server-destination-gcp 4.0.0 → 4.0.1-next-1778068549946

package/README.md

@@ -76,43 +76,95 @@ const { elb } = await startFlow({
 | ----------- | ----------------- | ------------------------------------------------ | -------- | ------------------------------------------ |
 | `client`    | `BigQuery`        | Google Cloud BigQuery client instance            | Yes      | `new BigQuery({ projectId, keyFilename })` |
 | `projectId` | `string`          | Google Cloud Project ID                          | Yes      | `'my-gcp-project'`                         |
-| `datasetId` | `string`          | BigQuery dataset ID where events will be stored  | Yes      | `'walker_events'`                          |
-| `tableId`   | `string`          | BigQuery table ID for event storage              | Yes      | `'events'`                                 |
-| `location`  | `string`          | Geographic location for the BigQuery dataset     | No       | `'US'`                                     |
+| `datasetId` | `string`          | BigQuery dataset ID where events will be stored  | No       | `'walkerOS'` (default)                     |
+| `tableId`   | `string`          | BigQuery table ID for event storage              | No       | `'events'` (default)                       |
+| `location`  | `string`          | Geographic location for the BigQuery dataset     | No       | `'EU'` (default)                           |
 | `bigquery`  | `BigQueryOptions` | Additional BigQuery client configuration options | No       | `{ keyFilename: "path/to/key.json" }`      |
 
-## Table Schema
+## Setup (one-time provisioning)
 
-By default, the destination sends the full walkerOS event. Create the table
-with:
-
-```sql
-CREATE TABLE IF NOT EXISTS `YOUR_PROJECT.walkeros.events` (
-  timestamp TIMESTAMP,
-  createdAt TIMESTAMP,
-  name STRING,
-  id STRING,
-  entity STRING,
-  action STRING,
-  trigger STRING,
-  `group` STRING,
-  timing FLOAT64,
-  count INT64,
-  data STRING,
-  context STRING,
-  globals STRING,
-  custom STRING,
-  user STRING,
-  nested STRING,
-  consent STRING,
-  version STRING,
-  source STRING
-);
+```bash
+walkeros setup destination.bigquery
 ```
 
-Object and array fields (`data`, `context`, `globals`, etc.) are JSON
-stringified. For custom schemas using the `data` mapping config, see the
-[full documentation](https://www.walkeros.io/docs/destinations/server/gcp).
+Output: `setup: ok destination.bigquery` plus a JSON line with
+`{ datasetCreated, tableCreated }`. Idempotent, safe to re-run.
+
+IAM the operator service account needs:
+
+- `bigquery.datasets.create`
+- `bigquery.tables.create`
+- `bigquery.datasets.get` (for drift detection)
+- `bigquery.tables.get`
+
+`config.setup`:
+
+- `false` (default): no provisioning. Operator must run setup once explicitly.
+- `true`: provision with defaults, `walkerOS` dataset, `events` table, `EU`
+  location, `PHYSICAL` storage billing, day partitioning on `timestamp`,
+  clustering on `(name, entity, action)`.
+- `{ ...overrides }`: object form to override any default. See the `Setup`
+  interface in `src/bigquery/types/index.ts`.
+
+If an existing table's partitioning, clustering, or schema differs from the
+declared configuration, setup logs `WARN setup.drift {...}` and continues. No
+auto-mutation.
+
+## Storage Write API (data plane)
+
+The destination uses BigQuery's Storage Write API for data ingestion.
+
+Note: the upstream `@google-cloud/bigquery-storage` package self-marks as
+`EXPERIMENTAL` (subject to change). Pinned at `^5.1.0`.
+
+## Batching
+
+`pushBatch` is implemented. Set the collector's `batch: <ms>` mapping setting to
+enable batching:
+
+```jsonc
+{
+  "destinations": {
+    "bigquery": {
+      "package": "@walkeros/server-destination-gcp",
+      "config": {
+        "settings": { "projectId": "..." },
+        "setup": true,
+        "mapping": { "batch": 1000 },
+      },
+    },
+  },
+}
+```
+
+`batch: 1000` (1 second) is a reasonable default for analytics workloads. All
+events buffered within the window flush as a single `appendRows` call.
+
+## Table Schema
+
+The default 15-column schema (walkerOS Event v4 canonical order):
+
+| Column      | Type      | Mode     |
+| ----------- | --------- | -------- |
+| `name`      | STRING    | REQUIRED |
+| `data`      | JSON      | NULLABLE |
+| `context`   | JSON      | NULLABLE |
+| `globals`   | JSON      | NULLABLE |
+| `custom`    | JSON      | NULLABLE |
+| `user`      | JSON      | NULLABLE |
+| `nested`    | JSON      | NULLABLE |
+| `consent`   | JSON      | NULLABLE |
+| `id`        | STRING    | NULLABLE |
+| `trigger`   | STRING    | NULLABLE |
+| `entity`    | STRING    | NULLABLE |
+| `action`    | STRING    | NULLABLE |
+| `timestamp` | TIMESTAMP | NULLABLE |
+| `timing`    | INT64     | NULLABLE |
+| `source`    | JSON      | NULLABLE |
+
+For custom schemas, override `config.setup.schema`. See the
+[full documentation](https://www.walkeros.io/docs/destinations/server/gcp) for
+custom mapping examples.
 
 ## Type Definitions
 

package/dist/dev.d.mts

@@ -1,4 +1,6 @@
 import * as _walkeros_core_dev from '@walkeros/core/dev';
+import * as _google_cloud_bigquery_storage from '@google-cloud/bigquery-storage';
+import { managedwriter } from '@google-cloud/bigquery-storage';
 import { DestinationServer } from '@walkeros/server-core';
 import { BigQuery } from '@google-cloud/bigquery';
 import { Flow, Hint } from '@walkeros/core';
@@ -14,6 +16,10 @@ declare namespace schemas {
 
 interface Env extends DestinationServer.Env {
     BigQuery?: typeof BigQuery;
+    WriterClient?: typeof managedwriter.WriterClient;
+    JSONWriter?: typeof managedwriter.JSONWriter;
+    adapt?: typeof _google_cloud_bigquery_storage.adapt;
+    managedwriterModule?: typeof managedwriter;
 }
 
 /**

package/dist/dev.d.ts

@@ -1,4 +1,6 @@
 import * as _walkeros_core_dev from '@walkeros/core/dev';
+import * as _google_cloud_bigquery_storage from '@google-cloud/bigquery-storage';
+import { managedwriter } from '@google-cloud/bigquery-storage';
 import { DestinationServer } from '@walkeros/server-core';
 import { BigQuery } from '@google-cloud/bigquery';
 import { Flow, Hint } from '@walkeros/core';
@@ -14,6 +16,10 @@ declare namespace schemas {
 
 interface Env extends DestinationServer.Env {
     BigQuery?: typeof BigQuery;
+    WriterClient?: typeof managedwriter.WriterClient;
+    JSONWriter?: typeof managedwriter.JSONWriter;
+    adapt?: typeof _google_cloud_bigquery_storage.adapt;
+    managedwriterModule?: typeof managedwriter;
 }
 
 /**