@walkeros/server-destination-gcp 4.0.0 → 4.0.1-next-1778183328892

package/README.md

@@ -4,16 +4,21 @@
   </a>
 </p>
 
-# GCP (BigQuery) Destination for walkerOS
+# GCP (BigQuery, Pub/Sub) Destination for walkerOS
 
 [Source Code](https://github.com/elbwalker/walkerOS/tree/main/packages/server/destinations/gcp)
 &bull;
 [NPM Package](https://www.npmjs.com/package/@walkeros/server-destination-gcp)
 
 walkerOS follows a **source → collector → destination** architecture. This GCP
-destination receives processed events from the walkerOS collector and streams
-them to Google BigQuery, enabling real-time data warehousing and analytics with
-Google Cloud's powerful data processing and machine learning capabilities.
+destination package ships two server-side sub-destinations:
+
+- **BigQuery**, for streaming events into a managed data warehouse for analytics
+  and machine learning workloads.
+- **Pub/Sub**, for publishing events to a topic for downstream fan-out, async
+  processing, or cross-region delivery.
+
+One npm install covers both. Pick the export that matches your use case.
 
 ## Installation
 
@@ -76,43 +81,255 @@ const { elb } = await startFlow({
 | ----------- | ----------------- | ------------------------------------------------ | -------- | ------------------------------------------ |
 | `client`    | `BigQuery`        | Google Cloud BigQuery client instance            | Yes      | `new BigQuery({ projectId, keyFilename })` |
 | `projectId` | `string`          | Google Cloud Project ID                          | Yes      | `'my-gcp-project'`                         |
-| `datasetId` | `string`          | BigQuery dataset ID where events will be stored  | Yes      | `'walker_events'`                          |
-| `tableId`   | `string`          | BigQuery table ID for event storage              | Yes      | `'events'`                                 |
-| `location`  | `string`          | Geographic location for the BigQuery dataset     | No       | `'US'`                                     |
+| `datasetId` | `string`          | BigQuery dataset ID where events will be stored  | No       | `'walkerOS'` (default)                     |
+| `tableId`   | `string`          | BigQuery table ID for event storage              | No       | `'events'` (default)                       |
+| `location`  | `string`          | Geographic location for the BigQuery dataset     | No       | `'EU'` (default)                           |
 | `bigquery`  | `BigQueryOptions` | Additional BigQuery client configuration options | No       | `{ keyFilename: "path/to/key.json" }`      |
 
+## Setup (one-time provisioning)
+
+```bash
+walkeros setup destination.bigquery
+```
+
+Output: `setup: ok destination.bigquery` plus a JSON line with
+`{ datasetCreated, tableCreated }`. Idempotent, safe to re-run.
+
+IAM the operator service account needs:
+
+- `bigquery.datasets.create`
+- `bigquery.tables.create`
+- `bigquery.datasets.get` (for drift detection)
+- `bigquery.tables.get`
+
+`config.setup`:
+
+- `false` (default): no provisioning. Operator must run setup once explicitly.
+- `true`: provision with defaults, `walkerOS` dataset, `events` table, `EU`
+  location, `PHYSICAL` storage billing, day partitioning on `timestamp`,
+  clustering on `(name, entity, action)`.
+- `{ ...overrides }`: object form to override any default. See the `Setup`
+  interface in `src/bigquery/types/index.ts`.
+
+If an existing table's partitioning, clustering, or schema differs from the
+declared configuration, setup logs `WARN setup.drift {...}` and continues. No
+auto-mutation.
+
+## Storage Write API (data plane)
+
+The destination uses BigQuery's Storage Write API for data ingestion.
+
+Note: the upstream `@google-cloud/bigquery-storage` package self-marks as
+`EXPERIMENTAL` (subject to change). Pinned at `^5.1.0`.
+
+## Batching
+
+`pushBatch` is implemented. Set the collector's `batch: <ms>` mapping setting to
+enable batching:
+
+```jsonc
+{
+  "destinations": {
+    "bigquery": {
+      "package": "@walkeros/server-destination-gcp",
+      "config": {
+        "settings": { "projectId": "..." },
+        "setup": true,
+        "mapping": { "batch": 1000 },
+      },
+    },
+  },
+}
+```
+
+`batch: 1000` (1 second) is a reasonable default for analytics workloads. All
+events buffered within the window flush as a single `appendRows` call.
+
 ## Table Schema
 
-By default, the destination sends the full walkerOS event. Create the table
-with:
-
-```sql
-CREATE TABLE IF NOT EXISTS `YOUR_PROJECT.walkeros.events` (
-  timestamp TIMESTAMP,
-  createdAt TIMESTAMP,
-  name STRING,
-  id STRING,
-  entity STRING,
-  action STRING,
-  trigger STRING,
-  `group` STRING,
-  timing FLOAT64,
-  count INT64,
-  data STRING,
-  context STRING,
-  globals STRING,
-  custom STRING,
-  user STRING,
-  nested STRING,
-  consent STRING,
-  version STRING,
-  source STRING
-);
+The default 15-column schema (walkerOS Event v4 canonical order):
+
+| Column      | Type      | Mode     |
+| ----------- | --------- | -------- |
+| `name`      | STRING    | REQUIRED |
+| `data`      | JSON      | NULLABLE |
+| `context`   | JSON      | NULLABLE |
+| `globals`   | JSON      | NULLABLE |
+| `custom`    | JSON      | NULLABLE |
+| `user`      | JSON      | NULLABLE |
+| `nested`    | JSON      | NULLABLE |
+| `consent`   | JSON      | NULLABLE |
+| `id`        | STRING    | NULLABLE |
+| `trigger`   | STRING    | NULLABLE |
+| `entity`    | STRING    | NULLABLE |
+| `action`    | STRING    | NULLABLE |
+| `timestamp` | TIMESTAMP | NULLABLE |
+| `timing`    | INT64     | NULLABLE |
+| `source`    | JSON      | NULLABLE |
+
+For custom schemas, override `config.setup.schema`. See the
+[full documentation](https://www.walkeros.io/docs/destinations/server/gcp) for
+custom mapping examples.
+
+## Pub/Sub
+
+Publishes walkerOS events to a Google Cloud Pub/Sub topic. Supports per-key
+ordering and dynamic attributes.
+
+### Quick Start
+
+```json
+{
+  "version": 4,
+  "flows": {
+    "default": {
+      "server": {},
+      "destinations": {
+        "pubsub": {
+          "package": "@walkeros/server-destination-gcp",
+          "code": "destinationPubSub",
+          "config": {
+            "settings": {
+              "projectId": "YOUR_PROJECT_ID",
+              "topic": "events"
+            }
+          }
+        }
+      }
+    }
+  }
+}
 ```
 
-Object and array fields (`data`, `context`, `globals`, etc.) are JSON
-stringified. For custom schemas using the `data` mapping config, see the
-[full documentation](https://www.walkeros.io/docs/destinations/server/gcp).
+Or programmatically:
+
+```typescript
+import { startFlow } from '@walkeros/collector';
+import { destinationPubSub } from '@walkeros/server-destination-gcp';
+
+const { elb } = await startFlow({
+  destinations: [
+    {
+      destination: destinationPubSub,
+      config: {
+        settings: {
+          projectId: 'YOUR_PROJECT_ID',
+          topic: 'events',
+        },
+      },
+    },
+  ],
+});
+```
+
+### Authentication
+
+Three modes (same chain as BigQuery):
+
+1. **Application Default Credentials (ADC)**: nothing to configure beyond
+   `projectId`. Works on GCP-native runtimes (Cloud Run, GKE) and locally with
+   `gcloud auth application-default login`.
+2. **Service account JSON**: pass the credentials object (or a JSON string) via
+   `settings.credentials`. The destination JSON-parses strings and forwards
+   `client_email` + `private_key` to the SDK.
+3. **Pre-configured client**: pass an existing `PubSub` instance as
+   `settings.client`. Useful for shared connections across destinations.
+
+When `settings.projectId` and `credentials.project_id` both resolve, the
+top-level `settings.projectId` wins.
+
+### Settings reference
+
+| Name          | Type                            | Description                                             | Required |
+| ------------- | ------------------------------- | ------------------------------------------------------- | -------- |
+| `projectId`   | `string`                        | Google Cloud Project ID                                 | Yes      |
+| `topic`       | `string`                        | Topic short name. SDK builds the full resource path.    | Yes      |
+| `credentials` | `string \| ServiceAccountCreds` | Service account credentials, JSON string or object      | No       |
+| `apiEndpoint` | `string`                        | Override Pub/Sub endpoint (use for the local emulator)  | No       |
+| `orderingKey` | `Mapping.Value`                 | Default ordering-key mapping. Truthy enables ordering.  | No       |
+| `attributes`  | `Mapping.Map`                   | Default per-event attributes merged into every publish  | No       |
+| `client`      | `PubSub`                        | Pre-configured SDK instance, bypasses constructor build | No       |
+
+### Mapping reference
+
+Per-rule overrides under `mapping.<entity>.<action>.settings`:
+
+| Name          | Type            | Description                                          |
+| ------------- | --------------- | ---------------------------------------------------- |
+| `topic`       | `string`        | Override the destination default topic for this rule |
+| `orderingKey` | `Mapping.Value` | Per-rule ordering-key path, overrides settings       |
+| `attributes`  | `Mapping.Map`   | Per-rule attributes merged on top of settings        |
+
+### Ordering
+
+`orderingKey` is a `Mapping.Value` resolved per event (for example `'user.id'`).
+When the resolved key is truthy, the topic handle is built with
+`messageOrdering: true` and the publish carries the resolved key. Pub/Sub
+guarantees per-key ordering for messages published to the same region.
+
+If a publish fails for an ordered key, Pub/Sub permanently halts subsequent
+publishes on that key until `topic.resumePublishing(key)` is called. The
+destination handles this automatically: on publish failure for an ordered key,
+it calls `resumePublishing` immediately and re-throws so the operator sees the
+error.
+
+### Setup (one-time provisioning)
+
+```bash
+walkeros setup destination.<id>
+```
+
+Provisions the topic if it does not already exist. Idempotent, safe to re-run.
+Defaults:
+
+| Field                                            | Value                                       |
+| ------------------------------------------------ | ------------------------------------------- |
+| `messageStoragePolicy.allowedPersistenceRegions` | `['eu-west1', 'eu-west3', 'eu-west4']` (EU) |
+| `messageRetentionDuration`                       | (unset, project default)                    |
+| `kmsKeyName`                                     | (unset)                                     |
+| `labels`                                         | (unset)                                     |
+
+If your project has an org policy restricting regions, override
+`messageStoragePolicy.allowedPersistenceRegions` accordingly.
+
+IAM the operator service account needs:
+
+- `pubsub.topics.create`
+- `pubsub.topics.get` (for drift detection)
+- `pubsub.topics.update` (only if you choose to mutate; the destination only
+  warns on drift, never auto-mutates)
+
+The runtime service account that publishes events needs `roles/pubsub.publisher`
+on the topic.
+
+If the existing topic's storage policy, retention, KMS key, or labels differ
+from the declared configuration, setup logs `WARN setup.drift {...}` and
+continues. Migrations are an operator decision.
+
+Subscription provisioning is owned by the Pub/Sub source, not the destination.
+
+### Emulator
+
+The official Pub/Sub emulator runs locally for development:
+
+```bash
+gcloud beta emulators pubsub start --host-port=localhost:8085
+export PUBSUB_EMULATOR_HOST=localhost:8085
+```
+
+The SDK automatically picks up `PUBSUB_EMULATOR_HOST`. To override explicitly,
+set `settings.apiEndpoint`.
+
+### Troubleshooting
+
+- **NOT_FOUND on publish**: the topic does not exist. The destination logs
+  `Pub/Sub topic "<topic>" not found in project "<projectId>". Run "walkeros setup destination.<id>" to create it.`
+  Run setup once.
+- **PERMISSION_DENIED / UNAUTHENTICATED**: the runtime service account lacks
+  `roles/pubsub.publisher` on the topic, or ADC is not configured.
+- **Ordering stuck**: a previous publish for an ordering key failed. The
+  destination calls `resumePublishing` automatically on failure; if you observe
+  sustained stalls, check publish-side error logs.
 
 ## Type Definitions
 

package/dist/dev.d.mts

@@ -1,19 +1,27 @@
 import * as _walkeros_core_dev from '@walkeros/core/dev';
+import * as _google_cloud_bigquery_storage from '@google-cloud/bigquery-storage';
+import { managedwriter } from '@google-cloud/bigquery-storage';
 import { DestinationServer } from '@walkeros/server-core';
 import { BigQuery } from '@google-cloud/bigquery';
 import { Flow, Hint } from '@walkeros/core';
 
+declare const settings$1: _walkeros_core_dev.JSONSchema;
+declare const mapping$1: _walkeros_core_dev.JSONSchema;
+
 declare const settings: _walkeros_core_dev.JSONSchema;
 declare const mapping: _walkeros_core_dev.JSONSchema;
+declare const setup: _walkeros_core_dev.JSONSchema;
 
-declare const schemas_mapping: typeof mapping;
-declare const schemas_settings: typeof settings;
 declare namespace schemas {
-  export { schemas_mapping as mapping, schemas_settings as settings };
+  export { mapping$1 as mapping, mapping as pubsubMapping, settings as pubsubSettings, setup as pubsubSetup, settings$1 as settings };
 }
 
 interface Env extends DestinationServer.Env {
     BigQuery?: typeof BigQuery;
+    WriterClient?: typeof managedwriter.WriterClient;
+    JSONWriter?: typeof managedwriter.JSONWriter;
+    adapt?: typeof _google_cloud_bigquery_storage.adapt;
+    managedwriterModule?: typeof managedwriter;
 }
 
 /**
@@ -23,21 +31,85 @@ interface Env extends DestinationServer.Env {
  * to actual GCP infrastructure.
  */
 declare const push: Env;
+declare const simulation$1: string[];
+
+declare const env$1_push: typeof push;
+declare namespace env$1 {
+  export { env$1_push as push, simulation$1 as simulation };
+}
+
+declare const pageView$1: Flow.StepExample;
+declare const purchase: Flow.StepExample;
+
+declare const step$1_purchase: typeof purchase;
+declare namespace step$1 {
+  export { pageView$1 as pageView, step$1_purchase as purchase };
+}
+
+declare namespace index$1 {
+  export { env$1 as env, step$1 as step };
+}
+
+/**
+ * Example environment metadata for GCP Pub/Sub destination.
+ *
+ * Tests substitute the real SDK via `jest.mock('@google-cloud/pubsub')`,
+ * which is the recommended pattern: imports of `@google-cloud/pubsub` get
+ * replaced module-wide, no env-injection plumbing required at the call site.
+ *
+ * The `simulation` list documents which globals the destination touches
+ * during a simulated run, used by the simulator to know what to stub.
+ */
 declare const simulation: string[];
 
-declare const env_push: typeof push;
 declare const env_simulation: typeof simulation;
 declare namespace env {
-  export { env_push as push, env_simulation as simulation };
+  export { env_simulation as simulation };
 }
 
+/**
+ * Default publish, no mapping. Settings.topic = 'events'. The destination
+ * constructs a topic handle inline with messageOrdering=false (no ordering
+ * key resolved) and publishes the full event JSON as the message body.
+ */
+declare const defaultPush: Flow.StepExample;
+/**
+ * Per-rule topic override. Mapping carries `settings.topic = 'orders'`
+ * which routes this rule to a dedicated topic.
+ */
+declare const mappedTopic: Flow.StepExample;
+/**
+ * Per-rule ordering key resolved from the event. `mapping.orderingKey =
+ * 'user.id'` is a Mapping.Value that resolves to the event's user.id. The
+ * topic handle is constructed with messageOrdering=true and the publish
+ * carries the resolved orderingKey.
+ */
+declare const mappedOrderingKey: Flow.StepExample;
+/**
+ * Per-rule attributes. `mapping.attributes = { entity: 'entity', action:
+ * 'action' }` is a Mapping.Map resolved per event. Each value is a path
+ * resolved against the event.
+ */
+declare const mappedAttributes: Flow.StepExample;
+/**
+ * Mapped data. `mapping.data` rewrites the publish body. The rewritten
+ * object replaces the event as the message data.
+ */
+declare const mappedData: Flow.StepExample;
+/**
+ * Page view, no rule. Verifies free-form publish on a non-ordered topic
+ * for an event with no special mapping.
+ */
 declare const pageView: Flow.StepExample;
-declare const purchase: Flow.StepExample;
 
+declare const step_defaultPush: typeof defaultPush;
+declare const step_mappedAttributes: typeof mappedAttributes;
+declare const step_mappedData: typeof mappedData;
+declare const step_mappedOrderingKey: typeof mappedOrderingKey;
+declare const step_mappedTopic: typeof mappedTopic;
 declare const step_pageView: typeof pageView;
-declare const step_purchase: typeof purchase;
 declare namespace step {
-  export { step_pageView as pageView, step_purchase as purchase };
+  export { step_defaultPush as defaultPush, step_mappedAttributes as mappedAttributes, step_mappedData as mappedData, step_mappedOrderingKey as mappedOrderingKey, step_mappedTopic as mappedTopic, step_pageView as pageView };
 }
 
 declare const index_env: typeof env;
@@ -48,4 +120,4 @@ declare namespace index {
 
 declare const hints: Hint.Hints;
 
-export { index as examples, hints, schemas };
+export { index$1 as examples, hints, index as pubsubExamples, schemas };

package/dist/dev.d.ts

@@ -1,19 +1,27 @@
 import * as _walkeros_core_dev from '@walkeros/core/dev';
+import * as _google_cloud_bigquery_storage from '@google-cloud/bigquery-storage';
+import { managedwriter } from '@google-cloud/bigquery-storage';
 import { DestinationServer } from '@walkeros/server-core';
 import { BigQuery } from '@google-cloud/bigquery';
 import { Flow, Hint } from '@walkeros/core';
 
+declare const settings$1: _walkeros_core_dev.JSONSchema;
+declare const mapping$1: _walkeros_core_dev.JSONSchema;
+
 declare const settings: _walkeros_core_dev.JSONSchema;
 declare const mapping: _walkeros_core_dev.JSONSchema;
+declare const setup: _walkeros_core_dev.JSONSchema;
 
-declare const schemas_mapping: typeof mapping;
-declare const schemas_settings: typeof settings;
 declare namespace schemas {
-  export { schemas_mapping as mapping, schemas_settings as settings };
+  export { mapping$1 as mapping, mapping as pubsubMapping, settings as pubsubSettings, setup as pubsubSetup, settings$1 as settings };
 }
 
 interface Env extends DestinationServer.Env {
     BigQuery?: typeof BigQuery;
+    WriterClient?: typeof managedwriter.WriterClient;
+    JSONWriter?: typeof managedwriter.JSONWriter;
+    adapt?: typeof _google_cloud_bigquery_storage.adapt;
+    managedwriterModule?: typeof managedwriter;
 }
 
 /**
@@ -23,21 +31,85 @@ interface Env extends DestinationServer.Env {
  * to actual GCP infrastructure.
  */
 declare const push: Env;
+declare const simulation$1: string[];
+
+declare const env$1_push: typeof push;
+declare namespace env$1 {
+  export { env$1_push as push, simulation$1 as simulation };
+}
+
+declare const pageView$1: Flow.StepExample;
+declare const purchase: Flow.StepExample;
+
+declare const step$1_purchase: typeof purchase;
+declare namespace step$1 {
+  export { pageView$1 as pageView, step$1_purchase as purchase };
+}
+
+declare namespace index$1 {
+  export { env$1 as env, step$1 as step };
+}
+
+/**
+ * Example environment metadata for GCP Pub/Sub destination.
+ *
+ * Tests substitute the real SDK via `jest.mock('@google-cloud/pubsub')`,
+ * which is the recommended pattern: imports of `@google-cloud/pubsub` get
+ * replaced module-wide, no env-injection plumbing required at the call site.
+ *
+ * The `simulation` list documents which globals the destination touches
+ * during a simulated run, used by the simulator to know what to stub.
+ */
 declare const simulation: string[];
 
-declare const env_push: typeof push;
 declare const env_simulation: typeof simulation;
 declare namespace env {
-  export { env_push as push, env_simulation as simulation };
+  export { env_simulation as simulation };
 }
 
+/**
+ * Default publish, no mapping. Settings.topic = 'events'. The destination
+ * constructs a topic handle inline with messageOrdering=false (no ordering
+ * key resolved) and publishes the full event JSON as the message body.
+ */
+declare const defaultPush: Flow.StepExample;
+/**
+ * Per-rule topic override. Mapping carries `settings.topic = 'orders'`
+ * which routes this rule to a dedicated topic.
+ */
+declare const mappedTopic: Flow.StepExample;
+/**
+ * Per-rule ordering key resolved from the event. `mapping.orderingKey =
+ * 'user.id'` is a Mapping.Value that resolves to the event's user.id. The
+ * topic handle is constructed with messageOrdering=true and the publish
+ * carries the resolved orderingKey.
+ */
+declare const mappedOrderingKey: Flow.StepExample;
+/**
+ * Per-rule attributes. `mapping.attributes = { entity: 'entity', action:
+ * 'action' }` is a Mapping.Map resolved per event. Each value is a path
+ * resolved against the event.
+ */
+declare const mappedAttributes: Flow.StepExample;
+/**
+ * Mapped data. `mapping.data` rewrites the publish body. The rewritten
+ * object replaces the event as the message data.
+ */
+declare const mappedData: Flow.StepExample;
+/**
+ * Page view, no rule. Verifies free-form publish on a non-ordered topic
+ * for an event with no special mapping.
+ */
 declare const pageView: Flow.StepExample;
-declare const purchase: Flow.StepExample;
 
+declare const step_defaultPush: typeof defaultPush;
+declare const step_mappedAttributes: typeof mappedAttributes;
+declare const step_mappedData: typeof mappedData;
+declare const step_mappedOrderingKey: typeof mappedOrderingKey;
+declare const step_mappedTopic: typeof mappedTopic;
 declare const step_pageView: typeof pageView;
-declare const step_purchase: typeof purchase;
 declare namespace step {
-  export { step_pageView as pageView, step_purchase as purchase };
+  export { step_defaultPush as defaultPush, step_mappedAttributes as mappedAttributes, step_mappedData as mappedData, step_mappedOrderingKey as mappedOrderingKey, step_mappedTopic as mappedTopic, step_pageView as pageView };
 }
 
 declare const index_env: typeof env;
@@ -48,4 +120,4 @@ declare namespace index {
 
 declare const hints: Hint.Hints;
 
-export { index as examples, hints, schemas };
+export { index$1 as examples, hints, index as pubsubExamples, schemas };