@walkeros/server-destination-gcp 0.0.0-next-20251219153324
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +135 -0
- package/dist/dev.d.mts +38 -0
- package/dist/dev.d.ts +38 -0
- package/dist/dev.js +1 -0
- package/dist/dev.js.map +1 -0
- package/dist/dev.mjs +1 -0
- package/dist/dev.mjs.map +1 -0
- package/dist/index.d.mts +59 -0
- package/dist/index.d.ts +59 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +1 -0
- package/dist/index.mjs.map +1 -0
- package/dist/schemas.d.mts +6 -0
- package/dist/schemas.d.ts +6 -0
- package/dist/schemas.js +1 -0
- package/dist/schemas.js.map +1 -0
- package/dist/schemas.mjs +1 -0
- package/dist/schemas.mjs.map +1 -0
- package/package.json +57 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 elbWalker GmbH
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
<p align="left">
|
|
2
|
+
<a href="https://www.walkeros.io">
|
|
3
|
+
<img title="elbwalker" src="https://www.walkeros.io/img/elbwalker_logo.png" width="256px"/>
|
|
4
|
+
</a>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
# GCP (BigQuery) Destination for walkerOS
|
|
8
|
+
|
|
9
|
+
[Source Code](https://github.com/elbwalker/walkerOS/tree/main/packages/server/destinations/gcp)
|
|
10
|
+
•
|
|
11
|
+
[NPM Package](https://www.npmjs.com/package/@walkeros/server-destination-gcp)
|
|
12
|
+
|
|
13
|
+
walkerOS follows a **source → collector → destination** architecture. This GCP
|
|
14
|
+
destination receives processed events from the walkerOS collector and streams
|
|
15
|
+
them to Google BigQuery, enabling real-time data warehousing and analytics with
|
|
16
|
+
Google Cloud's powerful data processing and machine learning capabilities.
|
|
17
|
+
|
|
18
|
+
## Installation
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
npm install @walkeros/server-destination-gcp
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Quick Start
|
|
25
|
+
|
|
26
|
+
Configure in your Flow JSON:
|
|
27
|
+
|
|
28
|
+
```json
|
|
29
|
+
{
|
|
30
|
+
"version": 1,
|
|
31
|
+
"flows": {
|
|
32
|
+
"default": {
|
|
33
|
+
"server": {},
|
|
34
|
+
"destinations": {
|
|
35
|
+
"bigquery": {
|
|
36
|
+
"package": "@walkeros/server-destination-gcp",
|
|
37
|
+
"config": {
|
|
38
|
+
"settings": {
|
|
39
|
+
"projectId": "YOUR_PROJECT_ID",
|
|
40
|
+
"datasetId": "YOUR_DATASET_ID",
|
|
41
|
+
"tableId": "YOUR_TABLE_ID"
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Or programmatically:
|
|
52
|
+
|
|
53
|
+
```typescript
|
|
54
|
+
import { startFlow } from '@walkeros/collector';
|
|
55
|
+
import { destinationBigQuery } from '@walkeros/server-destination-gcp';
|
|
56
|
+
|
|
57
|
+
const { elb } = await startFlow({
|
|
58
|
+
destinations: [
|
|
59
|
+
{
|
|
60
|
+
destination: destinationBigQuery,
|
|
61
|
+
config: {
|
|
62
|
+
settings: {
|
|
63
|
+
projectId: 'YOUR_PROJECT_ID',
|
|
64
|
+
datasetId: 'YOUR_DATASET_ID',
|
|
65
|
+
tableId: 'YOUR_TABLE_ID',
|
|
66
|
+
},
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
],
|
|
70
|
+
});
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Configuration
|
|
74
|
+
|
|
75
|
+
| Name | Type | Description | Required | Example |
|
|
76
|
+
| ----------- | ----------------- | ------------------------------------------------ | -------- | ------------------------------------------ |
|
|
77
|
+
| `client` | `BigQuery` | Google Cloud BigQuery client instance | Yes | `new BigQuery({ projectId, keyFilename })` |
|
|
78
|
+
| `projectId` | `string` | Google Cloud Project ID | Yes | `'my-gcp-project'` |
|
|
79
|
+
| `datasetId` | `string` | BigQuery dataset ID where events will be stored | Yes | `'walker_events'` |
|
|
80
|
+
| `tableId` | `string` | BigQuery table ID for event storage | Yes | `'events'` |
|
|
81
|
+
| `location` | `string` | Geographic location for the BigQuery dataset | No | `'US'` |
|
|
82
|
+
| `bigquery` | `BigQueryOptions` | Additional BigQuery client configuration options | No | `{ keyFilename: "path/to/key.json" }` |
|
|
83
|
+
|
|
84
|
+
## Table Schema
|
|
85
|
+
|
|
86
|
+
By default, the destination sends the full walkerOS event. Create the table
|
|
87
|
+
with:
|
|
88
|
+
|
|
89
|
+
```sql
|
|
90
|
+
CREATE TABLE IF NOT EXISTS `YOUR_PROJECT.walkeros.events` (
|
|
91
|
+
timestamp TIMESTAMP,
|
|
92
|
+
createdAt TIMESTAMP,
|
|
93
|
+
name STRING,
|
|
94
|
+
id STRING,
|
|
95
|
+
entity STRING,
|
|
96
|
+
action STRING,
|
|
97
|
+
trigger STRING,
|
|
98
|
+
`group` STRING,
|
|
99
|
+
timing FLOAT64,
|
|
100
|
+
count INT64,
|
|
101
|
+
data STRING,
|
|
102
|
+
context STRING,
|
|
103
|
+
globals STRING,
|
|
104
|
+
custom STRING,
|
|
105
|
+
user STRING,
|
|
106
|
+
nested STRING,
|
|
107
|
+
consent STRING,
|
|
108
|
+
version STRING,
|
|
109
|
+
source STRING
|
|
110
|
+
);
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Object and array fields (`data`, `context`, `globals`, etc.) are JSON
|
|
114
|
+
stringified. For custom schemas using the `data` mapping config, see the
|
|
115
|
+
[full documentation](https://www.walkeros.io/docs/destinations/server/gcp).
|
|
116
|
+
|
|
117
|
+
## Type Definitions
|
|
118
|
+
|
|
119
|
+
See [src/types/](./src/types/) for TypeScript interfaces.
|
|
120
|
+
|
|
121
|
+
## Related
|
|
122
|
+
|
|
123
|
+
- [Website Documentation](https://www.walkeros.io/docs/destinations/server/gcp/)
|
|
124
|
+
- [Destination Interface](../../../core/src/types/destination.ts)
|
|
125
|
+
|
|
126
|
+
## Contribute
|
|
127
|
+
|
|
128
|
+
Feel free to contribute by submitting an
|
|
129
|
+
[issue](https://github.com/elbwalker/walkerOS/issues), starting a
|
|
130
|
+
[discussion](https://github.com/elbwalker/walkerOS/discussions), or getting in
|
|
131
|
+
[contact](https://calendly.com/elb-alexander/30min).
|
|
132
|
+
|
|
133
|
+
## License
|
|
134
|
+
|
|
135
|
+
This project is licensed under the MIT License.
|
package/dist/dev.d.mts
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import * as _walkeros_core_dev from '@walkeros/core/dev';
|
|
2
|
+
import { DestinationServer } from '@walkeros/server-core';
|
|
3
|
+
import { BigQuery } from '@google-cloud/bigquery';
|
|
4
|
+
|
|
5
|
+
declare const settings: _walkeros_core_dev.JSONSchema;
|
|
6
|
+
declare const mapping: _walkeros_core_dev.JSONSchema;
|
|
7
|
+
|
|
8
|
+
declare const schemas_mapping: typeof mapping;
|
|
9
|
+
declare const schemas_settings: typeof settings;
|
|
10
|
+
declare namespace schemas {
|
|
11
|
+
export { schemas_mapping as mapping, schemas_settings as settings };
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
interface Env extends DestinationServer.Env {
|
|
15
|
+
BigQuery?: typeof BigQuery;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Standard mock environment for push operations
|
|
20
|
+
*
|
|
21
|
+
* Use this for testing BigQuery insert operations without connecting
|
|
22
|
+
* to actual GCP infrastructure.
|
|
23
|
+
*/
|
|
24
|
+
declare const push: Env;
|
|
25
|
+
declare const simulation: string[];
|
|
26
|
+
|
|
27
|
+
declare const env_push: typeof push;
|
|
28
|
+
declare const env_simulation: typeof simulation;
|
|
29
|
+
declare namespace env {
|
|
30
|
+
export { env_push as push, env_simulation as simulation };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
declare const index_env: typeof env;
|
|
34
|
+
declare namespace index {
|
|
35
|
+
export { index_env as env };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export { index as examples, schemas };
|
package/dist/dev.d.ts
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import * as _walkeros_core_dev from '@walkeros/core/dev';
|
|
2
|
+
import { DestinationServer } from '@walkeros/server-core';
|
|
3
|
+
import { BigQuery } from '@google-cloud/bigquery';
|
|
4
|
+
|
|
5
|
+
declare const settings: _walkeros_core_dev.JSONSchema;
|
|
6
|
+
declare const mapping: _walkeros_core_dev.JSONSchema;
|
|
7
|
+
|
|
8
|
+
declare const schemas_mapping: typeof mapping;
|
|
9
|
+
declare const schemas_settings: typeof settings;
|
|
10
|
+
declare namespace schemas {
|
|
11
|
+
export { schemas_mapping as mapping, schemas_settings as settings };
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
interface Env extends DestinationServer.Env {
|
|
15
|
+
BigQuery?: typeof BigQuery;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Standard mock environment for push operations
|
|
20
|
+
*
|
|
21
|
+
* Use this for testing BigQuery insert operations without connecting
|
|
22
|
+
* to actual GCP infrastructure.
|
|
23
|
+
*/
|
|
24
|
+
declare const push: Env;
|
|
25
|
+
declare const simulation: string[];
|
|
26
|
+
|
|
27
|
+
declare const env_push: typeof push;
|
|
28
|
+
declare const env_simulation: typeof simulation;
|
|
29
|
+
declare namespace env {
|
|
30
|
+
export { env_push as push, env_simulation as simulation };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
declare const index_env: typeof env;
|
|
34
|
+
declare namespace index {
|
|
35
|
+
export { index_env as env };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export { index as examples, schemas };
|