@walkeros/server-destination-gcp 4.7.0-next-1790245733116 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -15
- package/dist/dev.d.mts +185 -37
- package/dist/dev.d.ts +185 -37
- package/dist/dev.js +1 -1
- package/dist/dev.js.map +1 -1
- package/dist/dev.mjs +1 -1
- package/dist/dev.mjs.map +1 -1
- package/dist/index.d.mts +157 -31
- package/dist/index.d.ts +157 -31
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/index.mjs.map +1 -1
- package/dist/walkerOS.json +5199 -22
- package/package.json +6 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,24 +1,48 @@
|
|
|
1
1
|
# @walkeros/server-destination-gcp
|
|
2
2
|
|
|
3
|
-
## 4.7.0
|
|
3
|
+
## 4.7.0
|
|
4
4
|
|
|
5
|
-
###
|
|
5
|
+
### Minor Changes
|
|
6
6
|
|
|
7
|
-
-
|
|
7
|
+
- 64b06de: Packages with several exports, such as the GCP and AWS sources and
|
|
8
|
+
destinations, now publish a settings schema per export. `walkeros validate`
|
|
9
|
+
and MCP `package_get` use the schema of the export a step imports. Pub/Sub,
|
|
10
|
+
BigQuery, SNS and SQS steps pinned to this release or later are checked
|
|
11
|
+
instead of skipped; older pins still report a skip.
|
|
12
|
+
- 74821ed: Simulate records every vendor call through one recorder, `observeEnv`
|
|
13
|
+
from `@walkeros/core`, and runs offline: BigQuery, Data Manager and the
|
|
14
|
+
Sheets, GCS and S3 stores reach the network through `env`. Client and `Env`
|
|
15
|
+
types of BigQuery, Firehose, PostHog, SQS and Pub/Sub are structural.
|
|
16
|
+
`@walkeros/cli` exports `resolveExportName` and `selectDevExamples` and drops
|
|
17
|
+
`findExample` and `compareOutput`.
|
|
18
|
+
|
|
19
|
+
### Patch Changes
|
|
20
|
+
|
|
21
|
+
- 74821ed: Simulate starts only the simulated step, and a simulated transformer
|
|
22
|
+
continues through its `next`. Source simulations list the commands a source
|
|
23
|
+
issues as `elb` calls; `--page-url` sets the web page. Express flows need no
|
|
24
|
+
port. Packages with several exports use per-export examples and mocks, and a
|
|
25
|
+
destination without a mock env is refused instead of calling the vendor.
|
|
26
|
+
- Updated dependencies [74821ed]
|
|
8
27
|
- Updated dependencies [06b498a]
|
|
9
|
-
- Updated dependencies [
|
|
10
|
-
- Updated dependencies [
|
|
28
|
+
- Updated dependencies [74821ed]
|
|
29
|
+
- Updated dependencies [64b06de]
|
|
30
|
+
- Updated dependencies [4b4937f]
|
|
31
|
+
- Updated dependencies [74821ed]
|
|
32
|
+
- Updated dependencies [74821ed]
|
|
33
|
+
- Updated dependencies [74821ed]
|
|
11
34
|
- Updated dependencies [06b498a]
|
|
12
|
-
- Updated dependencies [
|
|
13
|
-
- Updated dependencies [
|
|
14
|
-
- Updated dependencies [
|
|
15
|
-
- Updated dependencies [
|
|
16
|
-
- Updated dependencies [
|
|
17
|
-
- Updated dependencies [
|
|
18
|
-
- Updated dependencies [
|
|
19
|
-
- Updated dependencies [
|
|
20
|
-
|
|
21
|
-
- @walkeros/
|
|
35
|
+
- Updated dependencies [74821ed]
|
|
36
|
+
- Updated dependencies [74821ed]
|
|
37
|
+
- Updated dependencies [74821ed]
|
|
38
|
+
- Updated dependencies [74821ed]
|
|
39
|
+
- Updated dependencies [74821ed]
|
|
40
|
+
- Updated dependencies [e860006]
|
|
41
|
+
- Updated dependencies [74821ed]
|
|
42
|
+
- Updated dependencies [74821ed]
|
|
43
|
+
- Updated dependencies [4f89234]
|
|
44
|
+
- @walkeros/core@4.7.0
|
|
45
|
+
- @walkeros/server-core@4.7.0
|
|
22
46
|
|
|
23
47
|
## 4.6.1
|
|
24
48
|
|
package/dist/dev.d.mts
CHANGED
|
@@ -1,41 +1,141 @@
|
|
|
1
1
|
import * as _walkeros_core_dev from '@walkeros/core/dev';
|
|
2
|
-
import * as _google_cloud_bigquery_storage from '@google-cloud/bigquery-storage';
|
|
3
|
-
import { managedwriter } from '@google-cloud/bigquery-storage';
|
|
4
2
|
import { DestinationServer } from '@walkeros/server-core';
|
|
5
|
-
import {
|
|
3
|
+
import { BigQueryOptions } from '@google-cloud/bigquery';
|
|
4
|
+
import { managedwriter } from '@google-cloud/bigquery-storage';
|
|
6
5
|
import { Flow, Hint } from '@walkeros/core';
|
|
6
|
+
import { ClientConfig } from '@google-cloud/pubsub';
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
8
|
+
type CallOptions = NonNullable<Parameters<managedwriter.WriterClient['getWriteStream']>[1]>;
|
|
9
|
+
type ClientOptions = NonNullable<ConstructorParameters<typeof managedwriter.WriterClient>[0]>;
|
|
10
|
+
type WriteStreamRequest = Parameters<managedwriter.WriterClient['getWriteStream']>[0];
|
|
11
|
+
type WriteStreamInfo = Awaited<ReturnType<managedwriter.WriterClient['getWriteStream']>>;
|
|
12
|
+
type TableSchema = NonNullable<WriteStreamInfo['tableSchema']>;
|
|
13
|
+
type ProtoDescriptor = ConstructorParameters<typeof managedwriter.JSONWriter>[0]['protoDescriptor'];
|
|
14
|
+
type RowList = Parameters<managedwriter.JSONWriter['appendRows']>[0];
|
|
15
|
+
type AppendResponse = Awaited<ReturnType<ReturnType<managedwriter.JSONWriter['appendRows']>['getResult']>>;
|
|
16
|
+
/** The `{ off }` disposable returned by `onConnectionError`. */
|
|
17
|
+
interface ConnectionListener {
|
|
18
|
+
off(): void;
|
|
19
|
+
}
|
|
20
|
+
/** The connection surface the writer uses: its stream id and error hook. */
|
|
21
|
+
interface WriteConnection {
|
|
22
|
+
getStreamId(): string;
|
|
23
|
+
onConnectionError(listener: (err: unknown) => void): ConnectionListener;
|
|
24
|
+
}
|
|
25
|
+
/** The fields of an append response that push and pushBatch read. */
|
|
26
|
+
type AppendOutcome = Pick<AppendResponse, 'rowErrors' | 'appendResult'>;
|
|
27
|
+
/** The row writer surface push and pushBatch use. */
|
|
28
|
+
interface RowWriter {
|
|
29
|
+
appendRows(rows: RowList): {
|
|
30
|
+
getResult(): Promise<AppendOutcome>;
|
|
31
|
+
};
|
|
32
|
+
close(): void;
|
|
33
|
+
}
|
|
34
|
+
/** The Storage Write client surface the writer uses. */
|
|
35
|
+
interface WriteClient<C extends WriteConnection> {
|
|
36
|
+
createStreamConnection(request: {
|
|
37
|
+
destinationTable: string;
|
|
38
|
+
streamId: string;
|
|
39
|
+
}): Promise<C>;
|
|
40
|
+
getWriteStream(request: WriteStreamRequest, options?: CallOptions): Promise<{
|
|
41
|
+
tableSchema?: TableSchema | null;
|
|
42
|
+
}>;
|
|
43
|
+
close(): void;
|
|
44
|
+
}
|
|
45
|
+
/** Converts the table schema into the writer's proto descriptor. */
|
|
46
|
+
interface SchemaAdapter {
|
|
47
|
+
convertStorageSchemaToProto2Descriptor(schema: TableSchema, scope: string): ProtoDescriptor;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* The three Storage Write pieces the writer is built from. The connection type
|
|
51
|
+
* ties a kit's client to the SAME kit's JSONWriter: the SDK JSONWriter only
|
|
52
|
+
* accepts the SDK's own StreamConnection.
|
|
53
|
+
*/
|
|
54
|
+
interface WriterKit<C extends WriteConnection> {
|
|
55
|
+
WriterClient: new (options: ClientOptions) => WriteClient<C>;
|
|
56
|
+
JSONWriter: new (args: {
|
|
57
|
+
connection: C;
|
|
58
|
+
protoDescriptor: ProtoDescriptor;
|
|
59
|
+
}) => RowWriter;
|
|
60
|
+
adapt: SchemaAdapter;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* The Storage Write API pieces a destination env may inject (tests, simulate).
|
|
64
|
+
* Without `WriterClient`, each missing piece falls back to the SDK, so runtime
|
|
65
|
+
* needs none of them. An injected `WriterClient` needs an injected
|
|
66
|
+
* `JSONWriter`: its connection cannot feed the SDK writer.
|
|
67
|
+
*/
|
|
68
|
+
interface WriterEnv {
|
|
69
|
+
WriterClient?: WriterKit<WriteConnection>['WriterClient'];
|
|
70
|
+
JSONWriter?: WriterKit<WriteConnection>['JSONWriter'];
|
|
71
|
+
adapt?: SchemaAdapter;
|
|
17
72
|
}
|
|
18
73
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
74
|
+
/** Table metadata fields the setup drift check reads. */
|
|
75
|
+
interface TableMetadataShape {
|
|
76
|
+
timePartitioning?: unknown;
|
|
77
|
+
clustering?: unknown;
|
|
78
|
+
schema?: unknown;
|
|
79
|
+
}
|
|
80
|
+
/** The table surface `walkeros setup` calls. */
|
|
81
|
+
interface QueryTable {
|
|
82
|
+
exists(): Promise<[boolean, ...unknown[]]>;
|
|
83
|
+
create(options: {
|
|
84
|
+
schema: {
|
|
85
|
+
fields: SetupSchemaField[];
|
|
86
|
+
};
|
|
87
|
+
timePartitioning: {
|
|
88
|
+
type: string;
|
|
89
|
+
field: string;
|
|
90
|
+
};
|
|
91
|
+
clustering: {
|
|
92
|
+
fields: string[];
|
|
93
|
+
};
|
|
94
|
+
}): Promise<unknown>;
|
|
95
|
+
getMetadata(): Promise<TableMetadataShape[]>;
|
|
96
|
+
}
|
|
97
|
+
/** The dataset surface `walkeros setup` calls. */
|
|
98
|
+
interface QueryDataset {
|
|
99
|
+
exists(): Promise<[boolean, ...unknown[]]>;
|
|
100
|
+
create(options: {
|
|
101
|
+
location: string;
|
|
102
|
+
storageBillingModel: string;
|
|
103
|
+
}): Promise<unknown>;
|
|
104
|
+
table(tableId: string): QueryTable;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* The query client surface this destination calls (dataset and table
|
|
108
|
+
* provisioning). A `BigQuery` from `@google-cloud/bigquery` satisfies it.
|
|
109
|
+
*/
|
|
110
|
+
interface QueryClient {
|
|
111
|
+
dataset(datasetId: string): QueryDataset;
|
|
112
|
+
}
|
|
113
|
+
/** Single column in the BQ schema (subset of @google-cloud/bigquery's TableField). */
|
|
114
|
+
interface SetupSchemaField {
|
|
115
|
+
name: string;
|
|
116
|
+
type: 'STRING' | 'JSON' | 'TIMESTAMP' | 'INT64' | 'BOOL' | 'FLOAT64';
|
|
117
|
+
mode?: 'NULLABLE' | 'REQUIRED' | 'REPEATED';
|
|
118
|
+
}
|
|
119
|
+
interface Env$1 extends DestinationServer.Env, WriterEnv {
|
|
120
|
+
BigQuery?: new (options?: BigQueryOptions) => QueryClient;
|
|
25
121
|
}
|
|
26
122
|
|
|
27
123
|
/**
|
|
28
124
|
* Standard mock environment for push operations
|
|
29
125
|
*
|
|
30
|
-
* Use this for testing BigQuery
|
|
126
|
+
* Use this for testing BigQuery Storage Write appends without connecting
|
|
31
127
|
* to actual GCP infrastructure.
|
|
32
128
|
*/
|
|
33
|
-
declare const push: Env;
|
|
129
|
+
declare const push$1: Env$1;
|
|
130
|
+
/**
|
|
131
|
+
* Every row goes through the writer's `appendRows`; the writer is built from
|
|
132
|
+
* `JSONWriter` in init, so the path continues on the constructed instance.
|
|
133
|
+
* Constructors are not recorded: their arguments carry credentials.
|
|
134
|
+
*/
|
|
34
135
|
declare const simulation$1: string[];
|
|
35
136
|
|
|
36
|
-
declare const env$1_push: typeof push;
|
|
37
137
|
declare namespace env$1 {
|
|
38
|
-
export {
|
|
138
|
+
export { push$1 as push, simulation$1 as simulation };
|
|
39
139
|
}
|
|
40
140
|
|
|
41
141
|
declare const pageView$1: Flow.StepExample;
|
|
@@ -46,25 +146,44 @@ declare namespace step$1 {
|
|
|
46
146
|
export { pageView$1 as pageView, step$1_purchase as purchase };
|
|
47
147
|
}
|
|
48
148
|
|
|
49
|
-
declare namespace
|
|
149
|
+
declare namespace bigqueryExamplesNs {
|
|
50
150
|
export { env$1 as env, step$1 as step };
|
|
51
151
|
}
|
|
52
152
|
|
|
153
|
+
/** The message `push` publishes. */
|
|
154
|
+
interface PublishMessage {
|
|
155
|
+
data: Buffer;
|
|
156
|
+
attributes?: Record<string, string>;
|
|
157
|
+
orderingKey?: string;
|
|
158
|
+
}
|
|
159
|
+
/** The part of a Pub/Sub topic handle `push` calls. */
|
|
160
|
+
interface TopicPublisher {
|
|
161
|
+
publishMessage(message: PublishMessage): Promise<string>;
|
|
162
|
+
resumePublishing(orderingKey: string): void;
|
|
163
|
+
}
|
|
53
164
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
* which is the recommended pattern: imports of `@google-cloud/pubsub` get
|
|
58
|
-
* replaced module-wide, no env-injection plumbing required at the call site.
|
|
59
|
-
*
|
|
60
|
-
* The `simulation` list documents which globals the destination touches
|
|
61
|
-
* during a simulated run, used by the simulator to know what to stub.
|
|
165
|
+
* The part of the Pub/Sub client `init`, `push` and `destroy` call. The SDK's
|
|
166
|
+
* `PubSub` satisfies it, and so does an injected mock (tests, simulate)
|
|
167
|
+
* without a cast. `setup` needs the admin surface and uses the SDK client.
|
|
62
168
|
*/
|
|
169
|
+
interface PubSubPublisher {
|
|
170
|
+
topic(name: string, options?: {
|
|
171
|
+
messageOrdering?: boolean;
|
|
172
|
+
}): TopicPublisher;
|
|
173
|
+
close(): Promise<void>;
|
|
174
|
+
}
|
|
175
|
+
interface Env extends DestinationServer.Env {
|
|
176
|
+
PubSub?: new (options?: ClientConfig) => PubSubPublisher;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
declare const push: Env;
|
|
180
|
+
/** The publish carries the request; `args[0]` is the message. */
|
|
63
181
|
declare const simulation: string[];
|
|
64
182
|
|
|
183
|
+
declare const env_push: typeof push;
|
|
65
184
|
declare const env_simulation: typeof simulation;
|
|
66
185
|
declare namespace env {
|
|
67
|
-
export { env_simulation as simulation };
|
|
186
|
+
export { env_push as push, env_simulation as simulation };
|
|
68
187
|
}
|
|
69
188
|
|
|
70
189
|
/**
|
|
@@ -112,12 +231,41 @@ declare namespace step {
|
|
|
112
231
|
export { step_defaultPush as defaultPush, step_mappedAttributes as mappedAttributes, step_mappedData as mappedData, step_mappedOrderingKey as mappedOrderingKey, step_mappedTopic as mappedTopic, step_pageView as pageView };
|
|
113
232
|
}
|
|
114
233
|
|
|
115
|
-
declare const
|
|
116
|
-
declare const
|
|
117
|
-
declare namespace
|
|
118
|
-
export {
|
|
234
|
+
declare const pubsubExamplesNs_env: typeof env;
|
|
235
|
+
declare const pubsubExamplesNs_step: typeof step;
|
|
236
|
+
declare namespace pubsubExamplesNs {
|
|
237
|
+
export { pubsubExamplesNs_env as env, pubsubExamplesNs_step as step };
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
declare const settings$1: _walkeros_core_dev.JSONSchema;
|
|
241
|
+
declare const mapping$1: _walkeros_core_dev.JSONSchema;
|
|
242
|
+
|
|
243
|
+
declare const settings: _walkeros_core_dev.JSONSchema;
|
|
244
|
+
declare const mapping: _walkeros_core_dev.JSONSchema;
|
|
245
|
+
declare const setup: _walkeros_core_dev.JSONSchema;
|
|
246
|
+
|
|
247
|
+
declare namespace schemas {
|
|
248
|
+
export { mapping$1 as mapping, mapping as pubsubMapping, settings as pubsubSettings, setup as pubsubSetup, settings$1 as settings };
|
|
119
249
|
}
|
|
120
250
|
|
|
121
251
|
declare const hints: Hint.Hints;
|
|
122
252
|
|
|
123
|
-
|
|
253
|
+
/** Dev examples per export, keyed by the export names in package.json. */
|
|
254
|
+
declare const exportExamples: {
|
|
255
|
+
destinationBigQuery: typeof bigqueryExamplesNs;
|
|
256
|
+
destinationPubSub: typeof pubsubExamplesNs;
|
|
257
|
+
};
|
|
258
|
+
/** Schemas per export, keyed by the export names in package.json. */
|
|
259
|
+
declare const exportSchemas: {
|
|
260
|
+
destinationBigQuery: {
|
|
261
|
+
settings: _walkeros_core_dev.JSONSchema;
|
|
262
|
+
mapping: _walkeros_core_dev.JSONSchema;
|
|
263
|
+
};
|
|
264
|
+
destinationPubSub: {
|
|
265
|
+
settings: _walkeros_core_dev.JSONSchema;
|
|
266
|
+
mapping: _walkeros_core_dev.JSONSchema;
|
|
267
|
+
setup: _walkeros_core_dev.JSONSchema;
|
|
268
|
+
};
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
export { bigqueryExamplesNs as examples, exportExamples, exportSchemas, hints, pubsubExamplesNs as pubsubExamples, schemas };
|
package/dist/dev.d.ts
CHANGED
|
@@ -1,41 +1,141 @@
|
|
|
1
1
|
import * as _walkeros_core_dev from '@walkeros/core/dev';
|
|
2
|
-
import * as _google_cloud_bigquery_storage from '@google-cloud/bigquery-storage';
|
|
3
|
-
import { managedwriter } from '@google-cloud/bigquery-storage';
|
|
4
2
|
import { DestinationServer } from '@walkeros/server-core';
|
|
5
|
-
import {
|
|
3
|
+
import { BigQueryOptions } from '@google-cloud/bigquery';
|
|
4
|
+
import { managedwriter } from '@google-cloud/bigquery-storage';
|
|
6
5
|
import { Flow, Hint } from '@walkeros/core';
|
|
6
|
+
import { ClientConfig } from '@google-cloud/pubsub';
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
8
|
+
type CallOptions = NonNullable<Parameters<managedwriter.WriterClient['getWriteStream']>[1]>;
|
|
9
|
+
type ClientOptions = NonNullable<ConstructorParameters<typeof managedwriter.WriterClient>[0]>;
|
|
10
|
+
type WriteStreamRequest = Parameters<managedwriter.WriterClient['getWriteStream']>[0];
|
|
11
|
+
type WriteStreamInfo = Awaited<ReturnType<managedwriter.WriterClient['getWriteStream']>>;
|
|
12
|
+
type TableSchema = NonNullable<WriteStreamInfo['tableSchema']>;
|
|
13
|
+
type ProtoDescriptor = ConstructorParameters<typeof managedwriter.JSONWriter>[0]['protoDescriptor'];
|
|
14
|
+
type RowList = Parameters<managedwriter.JSONWriter['appendRows']>[0];
|
|
15
|
+
type AppendResponse = Awaited<ReturnType<ReturnType<managedwriter.JSONWriter['appendRows']>['getResult']>>;
|
|
16
|
+
/** The `{ off }` disposable returned by `onConnectionError`. */
|
|
17
|
+
interface ConnectionListener {
|
|
18
|
+
off(): void;
|
|
19
|
+
}
|
|
20
|
+
/** The connection surface the writer uses: its stream id and error hook. */
|
|
21
|
+
interface WriteConnection {
|
|
22
|
+
getStreamId(): string;
|
|
23
|
+
onConnectionError(listener: (err: unknown) => void): ConnectionListener;
|
|
24
|
+
}
|
|
25
|
+
/** The fields of an append response that push and pushBatch read. */
|
|
26
|
+
type AppendOutcome = Pick<AppendResponse, 'rowErrors' | 'appendResult'>;
|
|
27
|
+
/** The row writer surface push and pushBatch use. */
|
|
28
|
+
interface RowWriter {
|
|
29
|
+
appendRows(rows: RowList): {
|
|
30
|
+
getResult(): Promise<AppendOutcome>;
|
|
31
|
+
};
|
|
32
|
+
close(): void;
|
|
33
|
+
}
|
|
34
|
+
/** The Storage Write client surface the writer uses. */
|
|
35
|
+
interface WriteClient<C extends WriteConnection> {
|
|
36
|
+
createStreamConnection(request: {
|
|
37
|
+
destinationTable: string;
|
|
38
|
+
streamId: string;
|
|
39
|
+
}): Promise<C>;
|
|
40
|
+
getWriteStream(request: WriteStreamRequest, options?: CallOptions): Promise<{
|
|
41
|
+
tableSchema?: TableSchema | null;
|
|
42
|
+
}>;
|
|
43
|
+
close(): void;
|
|
44
|
+
}
|
|
45
|
+
/** Converts the table schema into the writer's proto descriptor. */
|
|
46
|
+
interface SchemaAdapter {
|
|
47
|
+
convertStorageSchemaToProto2Descriptor(schema: TableSchema, scope: string): ProtoDescriptor;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* The three Storage Write pieces the writer is built from. The connection type
|
|
51
|
+
* ties a kit's client to the SAME kit's JSONWriter: the SDK JSONWriter only
|
|
52
|
+
* accepts the SDK's own StreamConnection.
|
|
53
|
+
*/
|
|
54
|
+
interface WriterKit<C extends WriteConnection> {
|
|
55
|
+
WriterClient: new (options: ClientOptions) => WriteClient<C>;
|
|
56
|
+
JSONWriter: new (args: {
|
|
57
|
+
connection: C;
|
|
58
|
+
protoDescriptor: ProtoDescriptor;
|
|
59
|
+
}) => RowWriter;
|
|
60
|
+
adapt: SchemaAdapter;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* The Storage Write API pieces a destination env may inject (tests, simulate).
|
|
64
|
+
* Without `WriterClient`, each missing piece falls back to the SDK, so runtime
|
|
65
|
+
* needs none of them. An injected `WriterClient` needs an injected
|
|
66
|
+
* `JSONWriter`: its connection cannot feed the SDK writer.
|
|
67
|
+
*/
|
|
68
|
+
interface WriterEnv {
|
|
69
|
+
WriterClient?: WriterKit<WriteConnection>['WriterClient'];
|
|
70
|
+
JSONWriter?: WriterKit<WriteConnection>['JSONWriter'];
|
|
71
|
+
adapt?: SchemaAdapter;
|
|
17
72
|
}
|
|
18
73
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
74
|
+
/** Table metadata fields the setup drift check reads. */
|
|
75
|
+
interface TableMetadataShape {
|
|
76
|
+
timePartitioning?: unknown;
|
|
77
|
+
clustering?: unknown;
|
|
78
|
+
schema?: unknown;
|
|
79
|
+
}
|
|
80
|
+
/** The table surface `walkeros setup` calls. */
|
|
81
|
+
interface QueryTable {
|
|
82
|
+
exists(): Promise<[boolean, ...unknown[]]>;
|
|
83
|
+
create(options: {
|
|
84
|
+
schema: {
|
|
85
|
+
fields: SetupSchemaField[];
|
|
86
|
+
};
|
|
87
|
+
timePartitioning: {
|
|
88
|
+
type: string;
|
|
89
|
+
field: string;
|
|
90
|
+
};
|
|
91
|
+
clustering: {
|
|
92
|
+
fields: string[];
|
|
93
|
+
};
|
|
94
|
+
}): Promise<unknown>;
|
|
95
|
+
getMetadata(): Promise<TableMetadataShape[]>;
|
|
96
|
+
}
|
|
97
|
+
/** The dataset surface `walkeros setup` calls. */
|
|
98
|
+
interface QueryDataset {
|
|
99
|
+
exists(): Promise<[boolean, ...unknown[]]>;
|
|
100
|
+
create(options: {
|
|
101
|
+
location: string;
|
|
102
|
+
storageBillingModel: string;
|
|
103
|
+
}): Promise<unknown>;
|
|
104
|
+
table(tableId: string): QueryTable;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* The query client surface this destination calls (dataset and table
|
|
108
|
+
* provisioning). A `BigQuery` from `@google-cloud/bigquery` satisfies it.
|
|
109
|
+
*/
|
|
110
|
+
interface QueryClient {
|
|
111
|
+
dataset(datasetId: string): QueryDataset;
|
|
112
|
+
}
|
|
113
|
+
/** Single column in the BQ schema (subset of @google-cloud/bigquery's TableField). */
|
|
114
|
+
interface SetupSchemaField {
|
|
115
|
+
name: string;
|
|
116
|
+
type: 'STRING' | 'JSON' | 'TIMESTAMP' | 'INT64' | 'BOOL' | 'FLOAT64';
|
|
117
|
+
mode?: 'NULLABLE' | 'REQUIRED' | 'REPEATED';
|
|
118
|
+
}
|
|
119
|
+
interface Env$1 extends DestinationServer.Env, WriterEnv {
|
|
120
|
+
BigQuery?: new (options?: BigQueryOptions) => QueryClient;
|
|
25
121
|
}
|
|
26
122
|
|
|
27
123
|
/**
|
|
28
124
|
* Standard mock environment for push operations
|
|
29
125
|
*
|
|
30
|
-
* Use this for testing BigQuery
|
|
126
|
+
* Use this for testing BigQuery Storage Write appends without connecting
|
|
31
127
|
* to actual GCP infrastructure.
|
|
32
128
|
*/
|
|
33
|
-
declare const push: Env;
|
|
129
|
+
declare const push$1: Env$1;
|
|
130
|
+
/**
|
|
131
|
+
* Every row goes through the writer's `appendRows`; the writer is built from
|
|
132
|
+
* `JSONWriter` in init, so the path continues on the constructed instance.
|
|
133
|
+
* Constructors are not recorded: their arguments carry credentials.
|
|
134
|
+
*/
|
|
34
135
|
declare const simulation$1: string[];
|
|
35
136
|
|
|
36
|
-
declare const env$1_push: typeof push;
|
|
37
137
|
declare namespace env$1 {
|
|
38
|
-
export {
|
|
138
|
+
export { push$1 as push, simulation$1 as simulation };
|
|
39
139
|
}
|
|
40
140
|
|
|
41
141
|
declare const pageView$1: Flow.StepExample;
|
|
@@ -46,25 +146,44 @@ declare namespace step$1 {
|
|
|
46
146
|
export { pageView$1 as pageView, step$1_purchase as purchase };
|
|
47
147
|
}
|
|
48
148
|
|
|
49
|
-
declare namespace
|
|
149
|
+
declare namespace bigqueryExamplesNs {
|
|
50
150
|
export { env$1 as env, step$1 as step };
|
|
51
151
|
}
|
|
52
152
|
|
|
153
|
+
/** The message `push` publishes. */
|
|
154
|
+
interface PublishMessage {
|
|
155
|
+
data: Buffer;
|
|
156
|
+
attributes?: Record<string, string>;
|
|
157
|
+
orderingKey?: string;
|
|
158
|
+
}
|
|
159
|
+
/** The part of a Pub/Sub topic handle `push` calls. */
|
|
160
|
+
interface TopicPublisher {
|
|
161
|
+
publishMessage(message: PublishMessage): Promise<string>;
|
|
162
|
+
resumePublishing(orderingKey: string): void;
|
|
163
|
+
}
|
|
53
164
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
* which is the recommended pattern: imports of `@google-cloud/pubsub` get
|
|
58
|
-
* replaced module-wide, no env-injection plumbing required at the call site.
|
|
59
|
-
*
|
|
60
|
-
* The `simulation` list documents which globals the destination touches
|
|
61
|
-
* during a simulated run, used by the simulator to know what to stub.
|
|
165
|
+
* The part of the Pub/Sub client `init`, `push` and `destroy` call. The SDK's
|
|
166
|
+
* `PubSub` satisfies it, and so does an injected mock (tests, simulate)
|
|
167
|
+
* without a cast. `setup` needs the admin surface and uses the SDK client.
|
|
62
168
|
*/
|
|
169
|
+
interface PubSubPublisher {
|
|
170
|
+
topic(name: string, options?: {
|
|
171
|
+
messageOrdering?: boolean;
|
|
172
|
+
}): TopicPublisher;
|
|
173
|
+
close(): Promise<void>;
|
|
174
|
+
}
|
|
175
|
+
interface Env extends DestinationServer.Env {
|
|
176
|
+
PubSub?: new (options?: ClientConfig) => PubSubPublisher;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
declare const push: Env;
|
|
180
|
+
/** The publish carries the request; `args[0]` is the message. */
|
|
63
181
|
declare const simulation: string[];
|
|
64
182
|
|
|
183
|
+
declare const env_push: typeof push;
|
|
65
184
|
declare const env_simulation: typeof simulation;
|
|
66
185
|
declare namespace env {
|
|
67
|
-
export { env_simulation as simulation };
|
|
186
|
+
export { env_push as push, env_simulation as simulation };
|
|
68
187
|
}
|
|
69
188
|
|
|
70
189
|
/**
|
|
@@ -112,12 +231,41 @@ declare namespace step {
|
|
|
112
231
|
export { step_defaultPush as defaultPush, step_mappedAttributes as mappedAttributes, step_mappedData as mappedData, step_mappedOrderingKey as mappedOrderingKey, step_mappedTopic as mappedTopic, step_pageView as pageView };
|
|
113
232
|
}
|
|
114
233
|
|
|
115
|
-
declare const
|
|
116
|
-
declare const
|
|
117
|
-
declare namespace
|
|
118
|
-
export {
|
|
234
|
+
declare const pubsubExamplesNs_env: typeof env;
|
|
235
|
+
declare const pubsubExamplesNs_step: typeof step;
|
|
236
|
+
declare namespace pubsubExamplesNs {
|
|
237
|
+
export { pubsubExamplesNs_env as env, pubsubExamplesNs_step as step };
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
declare const settings$1: _walkeros_core_dev.JSONSchema;
|
|
241
|
+
declare const mapping$1: _walkeros_core_dev.JSONSchema;
|
|
242
|
+
|
|
243
|
+
declare const settings: _walkeros_core_dev.JSONSchema;
|
|
244
|
+
declare const mapping: _walkeros_core_dev.JSONSchema;
|
|
245
|
+
declare const setup: _walkeros_core_dev.JSONSchema;
|
|
246
|
+
|
|
247
|
+
declare namespace schemas {
|
|
248
|
+
export { mapping$1 as mapping, mapping as pubsubMapping, settings as pubsubSettings, setup as pubsubSetup, settings$1 as settings };
|
|
119
249
|
}
|
|
120
250
|
|
|
121
251
|
declare const hints: Hint.Hints;
|
|
122
252
|
|
|
123
|
-
|
|
253
|
+
/** Dev examples per export, keyed by the export names in package.json. */
|
|
254
|
+
declare const exportExamples: {
|
|
255
|
+
destinationBigQuery: typeof bigqueryExamplesNs;
|
|
256
|
+
destinationPubSub: typeof pubsubExamplesNs;
|
|
257
|
+
};
|
|
258
|
+
/** Schemas per export, keyed by the export names in package.json. */
|
|
259
|
+
declare const exportSchemas: {
|
|
260
|
+
destinationBigQuery: {
|
|
261
|
+
settings: _walkeros_core_dev.JSONSchema;
|
|
262
|
+
mapping: _walkeros_core_dev.JSONSchema;
|
|
263
|
+
};
|
|
264
|
+
destinationPubSub: {
|
|
265
|
+
settings: _walkeros_core_dev.JSONSchema;
|
|
266
|
+
mapping: _walkeros_core_dev.JSONSchema;
|
|
267
|
+
setup: _walkeros_core_dev.JSONSchema;
|
|
268
|
+
};
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
export { bigqueryExamplesNs as examples, exportExamples, exportSchemas, hints, pubsubExamplesNs as pubsubExamples, schemas };
|
package/dist/dev.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var e,t=Object.defineProperty,r=Object.getOwnPropertyDescriptor,i=Object.getOwnPropertyNames,s=Object.prototype.hasOwnProperty,o=(e,r)=>{for(var i in r)t(e,i,{get:r[i],enumerable:!0})},a={};o(a,{examples:()=>O,hints:()=>ne,pubsubExamples:()=>L,schemas:()=>n}),module.exports=(e=a,((e,o,a,n)=>{if(o&&"object"==typeof o||"function"==typeof o)for(let l of i(o))s.call(e,l)||l===a||t(e,l,{get:()=>o[l],enumerable:!(n=r(o,l))||n.enumerable});return e})(t({},"__esModule",{value:!0}),e));var n={};o(n,{mapping:()=>g,pubsubMapping:()=>k,pubsubSettings:()=>w,pubsubSetup:()=>j,settings:()=>p});var l=require("@walkeros/core/dev"),c=require("@walkeros/core/dev"),u=c.z.object({client:c.z.any().describe("Google Cloud BigQuery client instance (like new BigQuery({ projectId, keyFilename }))").optional(),projectId:c.z.string().min(1).describe("Google Cloud Project ID (like my-gcp-project)"),datasetId:c.z.string().min(1).default("walkerOS").describe("BigQuery dataset ID where events will be stored (like walker_events)"),tableId:c.z.string().min(1).default("events").describe("BigQuery table ID for event storage (like events)"),location:c.z.string().default("EU").describe("Geographic location for the BigQuery dataset (like US)"),bigquery:c.z.any().describe('Additional BigQuery client configuration options (like { keyFilename: "path/to/key.json" })').optional()}),d=require("@walkeros/core/dev").z.object({}),p=(0,l.zodToSchema)(u),g=(0,l.zodToSchema)(d),m=require("@walkeros/core/dev"),b=require("@walkeros/core/dev"),h=b.z.object({client:b.z.any().describe("Google Cloud Pub/Sub client instance (like new PubSub({ projectId, keyFilename }))").optional(),projectId:b.z.string().min(1).describe("Google Cloud Project ID (like my-gcp-project)"),topic:b.z.string().min(1).describe("Pub/Sub topic short name (like events). The full resource name projects/<projectId>/topics/<topic> is built by the SDK."),credentials:b.z.any().describe("Service account credentials as a JSON string or an object with client_email and private_key. Default: Application Default Credentials (ADC). (deprecated: use config.credentials)").optional(),apiEndpoint:b.z.string().describe("Override Pub/Sub API endpoint. Useful for the local emulator (like localhost:8085).").optional(),orderingKey:b.z.any().describe("Mapping value resolved per-event. Truthy enables per-key ordering for the publish.").optional(),attributes:b.z.any().describe("Default per-event attribute map merged into every published message. Mapping.Map shape.").optional()}),y=require("@walkeros/core/dev"),v=y.z.object({topic:y.z.string().describe("Per-rule topic override. Falls back to settings.topic if absent.").optional(),orderingKey:y.z.any().describe("Per-rule ordering-key Mapping.Value. Overrides settings.orderingKey when set.").optional(),attributes:y.z.any().describe("Per-rule attribute Mapping.Map merged on top of settings.attributes.").optional()}),f=require("@walkeros/core/dev"),S=f.z.object({messageStoragePolicy:f.z.object({allowedPersistenceRegions:f.z.array(f.z.string()).describe("Geographic regions for at-rest message storage. Default: EU multi-region (eu-west1, eu-west3, eu-west4).")}).describe("Topic-level message storage policy.").optional(),messageRetentionDuration:f.z.object({seconds:f.z.number().describe("Retention window in seconds. Default: project default.")}).describe("Topic-level retention configuration.").optional(),kmsKeyName:f.z.string().describe("Customer-managed encryption key (CMEK) resource name. Optional.").optional(),labels:f.z.record(f.z.string(),f.z.string()).describe("Topic labels for organization and billing.").optional()}),w=(0,m.zodToSchema)(h),k=(0,m.zodToSchema)(v),j=(0,m.zodToSchema)(S),O={};o(O,{env:()=>z,step:()=>W});var z={};o(z,{push:()=>q,simulation:()=>M});var A=require("events"),C=[],P=null,D=null,E=null,T=null,I=class extends A.EventEmitter{streamId;constructor(e){super(),this.streamId=e}getStreamId(){return this.streamId}onConnectionError(e){return C.push({method:"onConnectionError",args:[]}),this.on("error",e),{off:()=>{this.off("error",e)}}}__emitConnectionError(e){this.emit("error",e)}},N=[],R={convertStorageSchemaToProto2Descriptor:(e,t)=>(C.push({method:"adapt.convertStorageSchemaToProto2Descriptor",args:[e,t]}),{name:"root",field:[]})},x={WriterClient:class{constructor(e){C.push({method:"WriterClient.ctor",args:[e]})}async createStreamConnection(e,t){if("DEFAULT"!==e.streamId)throw new Error(`mock createStreamConnection: expected streamId='DEFAULT', got ${JSON.stringify(e)}`);C.push({method:"createStreamConnection",args:[e,t]});const r=T;if(null!==r)throw T=null,r;const i=new I(`${e.destinationTable}/streams/_default`);return N.push(i),i}async getWriteStream(e,t){return C.push({method:"getWriteStream",args:[e,t]}),{tableSchema:{fields:[]}}}close(){C.push({method:"WriterClient.close",args:[]})}},JSONWriter:class{constructor(e){C.push({method:"JSONWriter.ctor",args:[e]})}appendRows(e,t){C.push({method:"appendRows",args:[e,t]});const r=D;if(null!==r)throw D=null,r;const i=P;P=null;const s=E;return E=null,{getResult:async()=>{if(null!==s)throw s;return{appendResult:{offset:{value:"0"}},rowErrors:i??[],writeStream:"projects/p/datasets/d/tables/t/streams/_default"}}}}close(){C.push({method:"JSONWriter.close",args:[]})}},DefaultStream:"DEFAULT"},J=()=>{};var q={get BigQuery(){return class{calls;options;constructor(e){this.options=e,this.calls=[]}dataset(e){return this.calls.push({method:"dataset",args:[e]}),this}table(e){return this.calls.push({method:"table",args:[e]}),this}async insert(e){return this.calls.push({method:"insert",args:[e]}),Promise.resolve()}get mockFn(){return J}}},get WriterClient(){return x.WriterClient},get JSONWriter(){return x.JSONWriter},get adapt(){return R},get managedwriterModule(){return x}},M=["BigQuery","WriterClient","JSONWriter","adapt"],W={};o(W,{pageView:()=>U,purchase:()=>G});var B=require("@walkeros/core");function _(e){return null==e?null:"object"==typeof e?Array.isArray(e)&&0===e.length?null:Array.isArray(e)||0!==Object.keys(e).length?JSON.stringify(e):null:JSON.stringify(e)}function Q(e){return function(e){return{name:e.name,data:_(e.data),context:_(e.context),globals:_(e.globals),custom:_(e.custom),user:_(e.user),nested:_(e.nested),consent:_(e.consent),id:e.id,trigger:e.trigger,entity:e.entity,action:e.action,timestamp:1e3*new Date(e.timestamp).getTime(),timing:e.timing,source:_(e.source)}}(e)}var F=(0,B.getEvent)("page view",{timestamp:1700001100,data:{title:"Documentation",url:"https://example.com/docs"},source:{type:"express",platform:"server"}}),U={title:"Page view",description:"A page view is appended as one row through the BigQuery Storage Write API JSONWriter. Nested objects/arrays in data, source, etc. are JSON-stringified by eventToRow.",in:F,mapping:void 0,out:[["appendRows",[Q(F)]]]},K=(0,B.getEvent)("order complete",{timestamp:1700001101,data:{id:"ORD-500",total:199.99,items:[{sku:"SKU-1",qty:2}]},source:{type:"express",platform:"server"}}),G={title:"Purchase",description:"An order event is appended as a single row through JSONWriter.appendRows. The entire nested data object (including arrays like items) is JSON-stringified into the data column via eventToRow().",in:K,mapping:void 0,out:[["appendRows",[Q(K)]]]},L={};o(L,{env:()=>V,step:()=>X});var V={};o(V,{simulation:()=>$});var $=["PubSub"],X={};o(X,{defaultPush:()=>te,mappedAttributes:()=>se,mappedData:()=>oe,mappedOrderingKey:()=>ie,mappedTopic:()=>re,pageView:()=>ae});var H=require("@walkeros/core"),Y=(0,H.getEvent)("order complete",{timestamp:1700001100,data:{id:"ORD-500",total:199.99,currency:"EUR"},user:{id:"usr-789"},source:{type:"express",platform:"server",trace:"1a2b3c4d5e6f70819a2b3c4d5e6f7081",count:1}}),Z=(0,H.getEvent)("page view",{timestamp:1700001101,data:{title:"Documentation",url:"https://example.com/docs"},source:{type:"express",platform:"server",trace:"9f8e7d6c5b4a39281706f5e4d3c2b1a0",count:1}});function ee(e){const t=e.source.platform??"default",r={...e,source:{...e.source,release:{[t]:"4.7.0-next-1790245733116"}}};return Buffer.from(JSON.stringify(r))}var te={title:"Default publish",description:"An event is published to the configured Pub/Sub topic with the full JSON event as the message body and no ordering key.",in:Y,out:[["topic","events",{messageOrdering:!1}],["publishMessage","events",{data:ee(Y)}]]},re={title:"Topic override",description:"A mapping rule overrides the destination default topic so order events are published to a dedicated topic.",in:Y,mapping:{settings:{topic:"orders"}},out:[["topic","orders",{messageOrdering:!1}],["publishMessage","orders",{data:ee(Y)}]]},ie={title:"Ordering key from user",description:"A mapping resolves the ordering key from event.user.id, enabling per-user ordering for this publish.",in:Y,mapping:{settings:{orderingKey:"user.id"}},out:[["topic","events",{messageOrdering:!0}],["publishMessage","events",{data:ee(Y),orderingKey:"usr-789"}]]},se={title:"Mapped attributes",description:"A mapping resolves a per-event attribute map, here exposing entity and action as Pub/Sub attributes for routing in subscribers.",in:Y,mapping:{settings:{attributes:{entity:"entity",action:"action"}}},out:[["topic","events",{messageOrdering:!1}],["publishMessage","events",{data:ee(Y),attributes:{entity:"order",action:"complete"}}]]},oe={title:"Mapped payload",description:"A data mapping transforms the event payload before publish. The rewritten object becomes the Pub/Sub message body.",in:Y,mapping:{data:{map:{order_id:"data.id",revenue:"data.total",currency:"data.currency"}}},out:[["topic","events",{messageOrdering:!1}],["publishMessage","events",{data:Buffer.from(JSON.stringify({order_id:"ORD-500",revenue:199.99,currency:"EUR"}))}]]},ae={title:"Page view",description:"A page view is published to the default topic with no ordering key, demonstrating the simplest publish path.",in:Z,out:[["topic","events",{messageOrdering:!1}],["publishMessage","events",{data:ee(Z)}]]},ne={"auth-methods":{text:'Supports three auth methods: SA key file via bigquery options ({ bigquery: { keyFilename: "./sa.json" } }), Application Default Credentials (automatic on GCP infrastructure — just provide projectId), or a pre-configured BigQuery client instance (client option). See settings schema for all options.',code:[{lang:"json",code:'{ "settings": { "projectId": "my-project", "bigquery": { "keyFilename": "./sa.json" } } }'},{lang:"json",code:'{ "settings": { "projectId": "my-project" } }'}]},"storage-format":{text:'All events are stored in a single table. Scalar fields (strings, numbers, booleans) are stored as-is. Nested objects and arrays are JSON-stringified automatically. Defaults: dataset "walkeros", table "events", location "EU" — all overridable via settings.'},"query-json-columns":{text:"Nested event fields (data, context, globals, user) are stored as JSON strings. Use BigQuery JSON_EXTRACT_SCALAR to query them. Field paths follow the walkerOS event structure.",code:[{lang:"sql",code:"SELECT JSON_EXTRACT_SCALAR(data, '$.total') AS total FROM `project.walkeros.events` WHERE entity = 'order'"}]},"troubleshoot-empty-table":{text:"Events not appearing? Common causes: projectId doesn't match GCP project, dataset doesn't exist yet (create via BigQuery console or bq CLI before first push), or SA key lacks bigquery.dataEditor role. The destination validates client/datasetId/tableId at push time but GCP permission errors surface from the BigQuery SDK."}};//# sourceMappingURL=dev.js.map
|
|
1
|
+
"use strict";var e,t=Object.defineProperty,r=Object.getOwnPropertyDescriptor,s=Object.getOwnPropertyNames,i=Object.prototype.hasOwnProperty,o=(e,r)=>{for(var s in r)t(e,s,{get:r[s],enumerable:!0})},a={};o(a,{examples:()=>n,exportExamples:()=>ue,exportSchemas:()=>pe,hints:()=>ce,pubsubExamples:()=>D,schemas:()=>le}),module.exports=(e=a,((e,o,a,n)=>{if(o&&"object"==typeof o||"function"==typeof o)for(let l of s(o))i.call(e,l)||l===a||t(e,l,{get:()=>o[l],enumerable:!(n=r(o,l))||n.enumerable});return e})(t({},"__esModule",{value:!0}),e));var n={};o(n,{env:()=>l,step:()=>k});var l={};o(l,{push:()=>S,simulation:()=>w});var c=require("events"),u=[],p=null,d=null,g=null,m=null,b=class extends c.EventEmitter{streamId;constructor(e){super(),this.streamId=e}getStreamId(){return this.streamId}onConnectionError(e){return u.push({method:"onConnectionError",args:[]}),this.on("error",e),{off:()=>{this.off("error",e)}}}__emitConnectionError(e){this.emit("error",e)}},h=[],y={convertStorageSchemaToProto2Descriptor:(e,t)=>(u.push({method:"adapt.convertStorageSchemaToProto2Descriptor",args:[e,t]}),{name:"root",field:[]})},v=class{constructor(e){u.push({method:"WriterClient.ctor",args:[e]})}async createStreamConnection(e,t){if("DEFAULT"!==e.streamId)throw new Error(`mock createStreamConnection: expected streamId='DEFAULT', got ${JSON.stringify(e)}`);u.push({method:"createStreamConnection",args:[e,t]});const r=m;if(null!==r)throw m=null,r;const s=new b(`${e.destinationTable}/streams/_default`);return h.push(s),s}async getWriteStream(e,t){return u.push({method:"getWriteStream",args:[e,t]}),{tableSchema:{fields:[]}}}close(){u.push({method:"WriterClient.close",args:[]})}},f=class{constructor(e){u.push({method:"JSONWriter.ctor",args:[e]})}appendRows(e,t){u.push({method:"appendRows",args:[e,t]});const r=d;if(null!==r)throw d=null,r;const s=p;p=null;const i=g;return g=null,{getResult:async()=>{if(null!==i)throw i;return{appendResult:{offset:{value:"0"}},rowErrors:s??[],writeStream:"projects/p/datasets/d/tables/t/streams/_default"}}}}close(){u.push({method:"JSONWriter.close",args:[]})}};var S={get BigQuery(){return class{calls;options;constructor(e){this.options=e,this.calls=[]}dataset(e){return this.calls.push({method:"dataset",args:[e]}),this}table(e){return this.calls.push({method:"table",args:[e]}),this}async exists(){return this.calls.push({method:"exists",args:[]}),[!0]}async create(e){this.calls.push({method:"create",args:[e]})}async getMetadata(){return this.calls.push({method:"getMetadata",args:[]}),[{}]}}},get WriterClient(){return v},get JSONWriter(){return f},get adapt(){return y}},w=["call:JSONWriter.appendRows"],k={};o(k,{pageView:()=>A,purchase:()=>C});var j=require("@walkeros/core");function O(e){return null==e?null:"object"==typeof e?Array.isArray(e)&&0===e.length?null:Array.isArray(e)||0!==Object.keys(e).length?JSON.stringify(e):null:JSON.stringify(e)}function z(e){return function(e){return{name:e.name,data:O(e.data),context:O(e.context),globals:O(e.globals),custom:O(e.custom),user:O(e.user),nested:O(e.nested),consent:O(e.consent),id:e.id,trigger:e.trigger,entity:e.entity,action:e.action,timestamp:1e3*new Date(e.timestamp).getTime(),timing:e.timing,source:O(e.source)}}(e)}var P=(0,j.getEvent)("page view",{timestamp:1700001100,data:{title:"Documentation",url:"https://example.com/docs"},source:{type:"express",platform:"server"}}),A={title:"Page view",description:"A page view is appended as one row through the BigQuery Storage Write API JSONWriter. Nested objects/arrays in data, source, etc. are JSON-stringified by eventToRow.",in:P,mapping:void 0,out:[["appendRows",[z(P)]]]},E=(0,j.getEvent)("order complete",{timestamp:1700001101,data:{id:"ORD-500",total:199.99,items:[{sku:"SKU-1",qty:2}]},source:{type:"express",platform:"server"}}),C={title:"Purchase",description:"An order event is appended as a single row through JSONWriter.appendRows. The entire nested data object (including arrays like items) is JSON-stringified into the data column via eventToRow().",in:E,mapping:void 0,out:[["appendRows",[z(E)]]]},D={};o(D,{env:()=>T,step:()=>R});var T={};o(T,{push:()=>x,simulation:()=>N});var I=class{constructor(e){this.name=e}name;publishMessage(e){return Promise.resolve("mock-message-id")}resumePublishing(e){}},x={PubSub:class{constructor(e){this.options=e}options;topic(e){return new I(e)}close(){return Promise.resolve()}}},N=["call:PubSub.topic.publishMessage"],R={};o(R,{defaultPush:()=>_,mappedAttributes:()=>K,mappedData:()=>U,mappedOrderingKey:()=>W,mappedTopic:()=>Q,pageView:()=>F});var M=require("@walkeros/core"),q=(0,M.getEvent)("order complete",{timestamp:1700001100,data:{id:"ORD-500",total:199.99,currency:"EUR"},user:{id:"usr-789"},source:{type:"express",platform:"server",trace:"1a2b3c4d5e6f70819a2b3c4d5e6f7081",count:1}}),J=(0,M.getEvent)("page view",{timestamp:1700001101,data:{title:"Documentation",url:"https://example.com/docs"},source:{type:"express",platform:"server",trace:"9f8e7d6c5b4a39281706f5e4d3c2b1a0",count:1}});function B(e){const t=e.source.platform??"default",r={...e,source:{...e.source,release:{[t]:"4.7.0"}}};return Buffer.from(JSON.stringify(r))}var _={title:"Default publish",description:"An event is published to the configured Pub/Sub topic with the full JSON event as the message body and no ordering key.",in:q,out:[["topic","events",{messageOrdering:!1}],["publishMessage","events",{data:B(q)}]]},Q={title:"Topic override",description:"A mapping rule overrides the destination default topic so order events are published to a dedicated topic.",in:q,mapping:{settings:{topic:"orders"}},out:[["topic","orders",{messageOrdering:!1}],["publishMessage","orders",{data:B(q)}]]},W={title:"Ordering key from user",description:"A mapping resolves the ordering key from event.user.id, enabling per-user ordering for this publish.",in:q,mapping:{settings:{orderingKey:"user.id"}},out:[["topic","events",{messageOrdering:!0}],["publishMessage","events",{data:B(q),orderingKey:"usr-789"}]]},K={title:"Mapped attributes",description:"A mapping resolves a per-event attribute map, here exposing entity and action as Pub/Sub attributes for routing in subscribers.",in:q,mapping:{settings:{attributes:{entity:"entity",action:"action"}}},out:[["topic","events",{messageOrdering:!1}],["publishMessage","events",{data:B(q),attributes:{entity:"order",action:"complete"}}]]},U={title:"Mapped payload",description:"A data mapping transforms the event payload before publish. The rewritten object becomes the Pub/Sub message body.",in:q,mapping:{data:{map:{order_id:"data.id",revenue:"data.total",currency:"data.currency"}}},out:[["topic","events",{messageOrdering:!1}],["publishMessage","events",{data:Buffer.from(JSON.stringify({order_id:"ORD-500",revenue:199.99,currency:"EUR"}))}]]},F={title:"Page view",description:"A page view is published to the default topic with no ordering key, demonstrating the simplest publish path.",in:J,out:[["topic","events",{messageOrdering:!1}],["publishMessage","events",{data:B(J)}]]},G=require("@walkeros/core/dev"),L=require("@walkeros/core/dev"),V=L.z.object({client:L.z.any().describe("Google Cloud BigQuery client instance (like new BigQuery({ projectId, keyFilename }))").optional(),projectId:L.z.string().min(1).describe("Google Cloud Project ID (like my-gcp-project)"),datasetId:L.z.string().min(1).default("walkerOS").describe("BigQuery dataset ID where events will be stored (like walker_events)"),tableId:L.z.string().min(1).default("events").describe("BigQuery table ID for event storage (like events)"),location:L.z.string().default("EU").describe("Geographic location for the BigQuery dataset (like US)"),bigquery:L.z.any().describe('Additional BigQuery client configuration options (like { keyFilename: "path/to/key.json" })').optional()}),$=require("@walkeros/core/dev").z.object({}),X=(0,G.zodToSchema)(V),H=(0,G.zodToSchema)($),Y=require("@walkeros/core/dev"),Z=require("@walkeros/core/dev"),ee=Z.z.object({client:Z.z.any().describe("Google Cloud Pub/Sub client instance (like new PubSub({ projectId, keyFilename }))").optional(),projectId:Z.z.string().min(1).describe("Google Cloud Project ID (like my-gcp-project)"),topic:Z.z.string().min(1).describe("Pub/Sub topic short name (like events). The full resource name projects/<projectId>/topics/<topic> is built by the SDK."),credentials:Z.z.any().describe("Service account credentials as a JSON string or an object with client_email and private_key. Default: Application Default Credentials (ADC). (deprecated: use config.credentials)").optional(),apiEndpoint:Z.z.string().describe("Override Pub/Sub API endpoint. Useful for the local emulator (like localhost:8085).").optional(),orderingKey:Z.z.any().describe("Mapping value resolved per-event. Truthy enables per-key ordering for the publish.").optional(),attributes:Z.z.any().describe("Default per-event attribute map merged into every published message. Mapping.Map shape.").optional()}),te=require("@walkeros/core/dev"),re=te.z.object({topic:te.z.string().describe("Per-rule topic override. Falls back to settings.topic if absent.").optional(),orderingKey:te.z.any().describe("Per-rule ordering-key Mapping.Value. Overrides settings.orderingKey when set.").optional(),attributes:te.z.any().describe("Per-rule attribute Mapping.Map merged on top of settings.attributes.").optional()}),se=require("@walkeros/core/dev"),ie=se.z.object({messageStoragePolicy:se.z.object({allowedPersistenceRegions:se.z.array(se.z.string()).describe("Geographic regions for at-rest message storage. Default: EU multi-region (eu-west1, eu-west3, eu-west4).")}).describe("Topic-level message storage policy.").optional(),messageRetentionDuration:se.z.object({seconds:se.z.number().describe("Retention window in seconds. Default: project default.")}).describe("Topic-level retention configuration.").optional(),kmsKeyName:se.z.string().describe("Customer-managed encryption key (CMEK) resource name. Optional.").optional(),labels:se.z.record(se.z.string(),se.z.string()).describe("Topic labels for organization and billing.").optional()}),oe=(0,Y.zodToSchema)(ee),ae=(0,Y.zodToSchema)(re),ne=(0,Y.zodToSchema)(ie),le={};o(le,{mapping:()=>H,pubsubMapping:()=>ae,pubsubSettings:()=>oe,pubsubSetup:()=>ne,settings:()=>X});var ce={"auth-methods":{text:'Supports three auth methods: SA key file via bigquery options ({ bigquery: { keyFilename: "./sa.json" } }), Application Default Credentials (automatic on GCP infrastructure — just provide projectId), or a pre-configured BigQuery client instance (client option). See settings schema for all options.',code:[{lang:"json",code:'{ "settings": { "projectId": "my-project", "bigquery": { "keyFilename": "./sa.json" } } }'},{lang:"json",code:'{ "settings": { "projectId": "my-project" } }'}]},"storage-format":{text:'All events are stored in a single table. Scalar fields (strings, numbers, booleans) are stored as-is. Nested objects and arrays are JSON-stringified automatically. Defaults: dataset "walkeros", table "events", location "EU" — all overridable via settings.'},"query-json-columns":{text:"Nested event fields (data, context, globals, user) are stored as JSON strings. Use BigQuery JSON_EXTRACT_SCALAR to query them. Field paths follow the walkerOS event structure.",code:[{lang:"sql",code:"SELECT JSON_EXTRACT_SCALAR(data, '$.total') AS total FROM `project.walkeros.events` WHERE entity = 'order'"}]},"troubleshoot-empty-table":{text:"Events not appearing? Common causes: projectId doesn't match GCP project, dataset doesn't exist yet (create via BigQuery console or bq CLI before first push), or SA key lacks bigquery.dataEditor role. The destination validates client/datasetId/tableId at push time but GCP permission errors surface from the BigQuery SDK."}},ue={destinationBigQuery:n,destinationPubSub:D},pe={destinationBigQuery:{settings:X,mapping:H},destinationPubSub:{settings:oe,mapping:ae,setup:ne}};//# sourceMappingURL=dev.js.map
|