ga4-export-fixer 0.9.0-dev.9 → 0.9.1-dev.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -29
- package/documentation.js +2 -2
- package/helpers/ga4Transforms.js +64 -4
- package/package.json +1 -1
- package/tables/ga4EventsEnhanced/columns/columnDescriptions.json +1 -0
- package/tables/ga4EventsEnhanced/columns/columnLineage.json +3 -0
- package/tables/ga4EventsEnhanced/columns/columnTypicalUse.json +1 -0
- package/tables/ga4EventsEnhanced/config.js +3 -2
- package/tables/ga4EventsEnhanced/index.js +145 -47
- package/tables/ga4EventsEnhanced/validation.js +8 -8
- package/utils.js +45 -35
package/README.md
CHANGED
|
@@ -16,6 +16,7 @@ The goal of the package is to **speed up development** when building data models
|
|
|
16
16
|
|
|
17
17
|
## Table of Contents
|
|
18
18
|
<!-- TOC -->
|
|
19
|
+
- [Blog Posts](#blog-posts)
|
|
19
20
|
- [Main Features](#main-features)
|
|
20
21
|
- [Planned Features](#planned-features)
|
|
21
22
|
- [Installation](#installation)
|
|
@@ -30,6 +31,10 @@ The goal of the package is to **speed up development** when building data models
|
|
|
30
31
|
- [License](#license)
|
|
31
32
|
<!-- /TOC -->
|
|
32
33
|
|
|
34
|
+
## Blog Posts
|
|
35
|
+
|
|
36
|
+
- [ga4-export-fixer: Getting started in Dataform](https://tanelytics.com/ga4-export-fixer-getting-started-in-dataform/)
|
|
37
|
+
|
|
33
38
|
## Main Features
|
|
34
39
|
|
|
35
40
|
<table>
|
|
@@ -44,96 +49,100 @@ The goal of the package is to **speed up development** when building data models
|
|
|
44
49
|
</td>
|
|
45
50
|
</tr>
|
|
46
51
|
<tr>
|
|
52
|
+
<td valign="top">
|
|
53
|
+
<b>🧬 Data Enrichments</b><br>
|
|
54
|
+
Join external lookup data (cohorts, product master, etc.) at row level or ecommerce item level via <code>enrichments</code>
|
|
55
|
+
</td>
|
|
47
56
|
<td valign="top">
|
|
48
57
|
<b>📐 Flexible Schema</b><br>
|
|
49
58
|
Keeps the flexible structure of the original export with key fields promoted to columns for better query performance; partitioning & clustering enabled
|
|
50
59
|
</td>
|
|
60
|
+
</tr>
|
|
61
|
+
<tr>
|
|
51
62
|
<td valign="top">
|
|
52
63
|
<b>🤖 AI Agent Ready</b><br>
|
|
53
64
|
Extensive table & column descriptions for AI agents and humans
|
|
54
65
|
</td>
|
|
55
|
-
</tr>
|
|
56
|
-
<tr>
|
|
57
66
|
<td valign="top">
|
|
58
67
|
<b>🔑 Session Identity Resolution</b><br>
|
|
59
68
|
<code>user_id</code> resolved per session; <code>merged_user_id</code> coalesces with <code>user_pseudo_id</code>
|
|
60
69
|
</td>
|
|
70
|
+
</tr>
|
|
71
|
+
<tr>
|
|
61
72
|
<td valign="top">
|
|
62
73
|
<b>📡 Session Traffic Sources</b><br>
|
|
63
74
|
<code>session_first_traffic_source</code> and <code>session_traffic_source_last_click</code> computed automatically, adjusting for sessions that span midnight
|
|
64
75
|
</td>
|
|
65
|
-
</tr>
|
|
66
|
-
<tr>
|
|
67
76
|
<td valign="top">
|
|
68
77
|
<b>📍 Landing Page Detection</b><br>
|
|
69
78
|
Derived per session from the first page where <code>entrances > 0</code>
|
|
70
79
|
</td>
|
|
80
|
+
</tr>
|
|
81
|
+
<tr>
|
|
71
82
|
<td valign="top">
|
|
72
83
|
<b>🔗 Page URL Parsing</b><br>
|
|
73
84
|
Parsed <code>hostname</code>, <code>path</code>, <code>query</code>, and <code>query_params</code> from <code>page_location</code>
|
|
74
85
|
</td>
|
|
75
|
-
</tr>
|
|
76
|
-
<tr>
|
|
77
86
|
<td valign="top">
|
|
78
87
|
<b>🛒 Ecommerce Data Fixes</b><br>
|
|
79
88
|
Nullifies placeholder <code>transaction_id</code>; corrects <code>purchase_revenue</code> bugs
|
|
80
89
|
</td>
|
|
90
|
+
</tr>
|
|
91
|
+
<tr>
|
|
81
92
|
<td valign="top">
|
|
82
93
|
<b>🏷️ Item List Attribution</b><br>
|
|
83
94
|
Attributes <code>item_list_name</code>, <code>item_list_id</code>, and <code>item_list_index</code> from item selection events to downstream ecommerce events
|
|
84
95
|
</td>
|
|
85
|
-
</tr>
|
|
86
|
-
<tr>
|
|
87
96
|
<td valign="top">
|
|
88
97
|
<b>⚙️ Event Parameter Handling</b><br>
|
|
89
98
|
Promote event params to columns; include or exclude by name
|
|
90
99
|
</td>
|
|
100
|
+
</tr>
|
|
101
|
+
<tr>
|
|
91
102
|
<td valign="top">
|
|
92
103
|
<b>📊 Session Parameters</b><br>
|
|
93
104
|
Promote selected event parameters as <code>session_params</code>
|
|
94
105
|
</td>
|
|
95
|
-
</tr>
|
|
96
|
-
<tr>
|
|
97
106
|
<td valign="top">
|
|
98
107
|
<b>⏱️ Custom Timestamp</b><br>
|
|
99
108
|
Use a custom event parameter as primary timestamp with automatic fallback
|
|
100
109
|
</td>
|
|
110
|
+
</tr>
|
|
111
|
+
<tr>
|
|
101
112
|
<td valign="top">
|
|
102
113
|
<b>🔒 Schema Lock</b><br>
|
|
103
114
|
Lock table schema to a specific GA4 export date to prevent schema drift
|
|
104
115
|
</td>
|
|
105
|
-
</tr>
|
|
106
|
-
<tr>
|
|
107
116
|
<td valign="top">
|
|
108
117
|
<b>✅ Data Freshness Tracking</b><br>
|
|
109
118
|
<code>data_is_final</code> flag and <code>export_type</code> label on every row
|
|
110
119
|
</td>
|
|
120
|
+
</tr>
|
|
121
|
+
<tr>
|
|
111
122
|
<td valign="top">
|
|
112
123
|
<b>🔍 Data Quality Assertions</b><br>
|
|
113
124
|
Built-in daily assertion reconciles sessions, events, and revenue between the enhanced table and raw export
|
|
114
125
|
</td>
|
|
115
|
-
</tr>
|
|
116
|
-
<tr>
|
|
117
126
|
<td valign="top">
|
|
118
127
|
<b>🔃 Selective Re-processing</b><br>
|
|
119
128
|
Re-process a date range without full table rebuild using <code>incrementalStartOverride</code> and <code>incrementalEndOverride</code>
|
|
120
129
|
</td>
|
|
130
|
+
</tr>
|
|
131
|
+
<tr>
|
|
121
132
|
<td valign="top">
|
|
122
133
|
<b>📑 Batch Processing</b><br>
|
|
123
134
|
Process large exports in smaller batches via <code>numberOfDaysToProcess</code>
|
|
124
135
|
</td>
|
|
125
|
-
</tr>
|
|
126
|
-
<tr>
|
|
127
136
|
<td valign="top">
|
|
128
137
|
<b>🕐 Timezone-Aware Datetime</b><br>
|
|
129
138
|
<code>event_datetime</code> converted to a configurable IANA timezone
|
|
130
139
|
</td>
|
|
140
|
+
</tr>
|
|
141
|
+
<tr>
|
|
131
142
|
<td valign="top">
|
|
132
143
|
<b>🧩 Custom Processing Steps</b><br>
|
|
133
144
|
Append user-defined CTEs via <code>customSteps</code> to derive new columns or join external tables
|
|
134
145
|
</td>
|
|
135
|
-
</tr>
|
|
136
|
-
<tr>
|
|
137
146
|
<td valign="top">
|
|
138
147
|
<b>🛡️ Zero Dependencies</b><br>
|
|
139
148
|
No additional external dependencies added to your Dataform repository
|
|
@@ -145,8 +154,8 @@ The goal of the package is to **speed up development** when building data models
|
|
|
145
154
|
|
|
146
155
|
Features under consideration for future releases:
|
|
147
156
|
|
|
148
|
-
- Data enrichment (item-level, session-level, event-level)
|
|
149
157
|
- Aggregated tables (ga4_session, ga4_ecommerce...)
|
|
158
|
+
- Roll-ups for creating a single ga4_events_enhanced table from multiple source exports
|
|
150
159
|
- Web and app specific default configurations
|
|
151
160
|
- Custom channel grouping
|
|
152
161
|
- Custom traffic source attribution
|
|
@@ -169,7 +178,7 @@ Include the package in the package.json file in your Dataform repository.
|
|
|
169
178
|
{
|
|
170
179
|
"dependencies": {
|
|
171
180
|
"@dataform/core": "3.0.42",
|
|
172
|
-
"ga4-export-fixer": "0.
|
|
181
|
+
"ga4-export-fixer": "0.9.0"
|
|
173
182
|
}
|
|
174
183
|
}
|
|
175
184
|
```
|
|
@@ -198,7 +207,8 @@ Create a new **ga4_events_enhanced** table using a **.js** file in your reposito
|
|
|
198
207
|
const { ga4EventsEnhanced } = require('ga4-export-fixer');
|
|
199
208
|
|
|
200
209
|
const config = {
|
|
201
|
-
|
|
210
|
+
// using hard-coded GA4 export path
|
|
211
|
+
sourceTable: '`project.analytics_12345.events_*`'
|
|
202
212
|
};
|
|
203
213
|
|
|
204
214
|
ga4EventsEnhanced.createTable(publish, config);
|
|
@@ -212,6 +222,7 @@ ga4EventsEnhanced.createTable(publish, config);
|
|
|
212
222
|
const { ga4EventsEnhanced } = require('ga4-export-fixer');
|
|
213
223
|
|
|
214
224
|
const config = {
|
|
225
|
+
// GA4 export path declared, using the table reference object
|
|
215
226
|
sourceTable: constants.GA4_TABLES.MY_GA4_EXPORT,
|
|
216
227
|
// use dataformTableConfig to make changes to the default Dataform table configuration
|
|
217
228
|
dataformTableConfig: {
|
|
@@ -290,7 +301,8 @@ js {
|
|
|
290
301
|
const { ga4EventsEnhanced } = require('ga4-export-fixer');
|
|
291
302
|
|
|
292
303
|
const config = {
|
|
293
|
-
|
|
304
|
+
// using hard-coded GA4 export path
|
|
305
|
+
sourceTable: '`project.analytics_12345.events_*`',
|
|
294
306
|
self: self(),
|
|
295
307
|
incremental: incremental()
|
|
296
308
|
};
|
|
@@ -534,13 +546,13 @@ For typical use cases this is the right tool; reach for `customSteps` only when
|
|
|
534
546
|
| Field | Type | Required | Description |
|
|
535
547
|
| --- | --- | --- | --- |
|
|
536
548
|
| `name` | string | Yes | Used in the generated `enrich_<name>` CTE name. Unique within `enrichments`. |
|
|
537
|
-
| `level` | `'
|
|
549
|
+
| `level` | `'row'` / `'item'` | No, defaults to `'row'` | Join grain. `'row'` joins external dim data onto each row of `enhanced_events` (any column on `enhanced_events` as the key). `'item'` joins external dim data onto each item inside the `items` array (any field on the items struct or any event_data column as the key). |
|
|
538
550
|
| `source` | Dataform ref / object / string | Yes | Source dim table. Inside an SQLX `js { }` block use `ref(...)`. From a `.js` definition file use a `{ schema, name }` ref object (resolved later via `ctx.ref()`) or a backtick-quoted ``` `project.dataset.table` ``` string for an external table. |
|
|
539
|
-
| `joinKey` | string / string[] | Yes |
|
|
551
|
+
| `joinKey` | string / string[] | Yes | For `level: 'row'`: column name(s) on `enhanced_events`. For `level: 'item'`: field name(s) on the items struct (e.g. `'item_id'`) or column name(s) on `event_data` (e.g. `'user_pseudo_id'`). Composite keys (array) compile to `USING(col1, col2, ...)`. |
|
|
540
552
|
| `columns` | string[] | Yes | Source columns to add to the output (excluding `joinKey`). Names matching existing columns are coalesced with the original (`coalesce(enrich.col, original)`) so missed JOINs fall back to the existing value. |
|
|
541
553
|
| `dedupe` | boolean | No, defaults to `false` | When `true`, wraps the source CTE in `qualify row_number() over (partition by <joinKey>) = 1` for non-unique-key dim sources. Non-deterministic which row wins; for strict needs, pre-aggregate in source SQL. |
|
|
542
554
|
|
|
543
|
-
**Coalesce-or-add semantics.** If an enrichment column name matches an existing column on `enhanced_events` (a column promoted via `eventParamsToColumns`, a package-generated column, or a default GA4 column from the export), the enrichment value is coalesced with the original: `coalesce(enrich_<name>.<col>, <original>) as <col>`. Rows where the JOIN matches get the enrichment value; rows where it misses fall back to the existing value rather than going NULL. If there is no overlap, the column is added as a plain `enrich_<name>.<col>`.
|
|
555
|
+
**Coalesce-or-add semantics.** If an enrichment column name matches an existing column on `enhanced_events` (a column promoted via `eventParamsToColumns`, a package-generated column, or a default GA4 column from the export), the enrichment value is coalesced with the original: `coalesce(enrich_<name>.<col>, <original>) as <col>`. Rows where the JOIN matches get the enrichment value; rows where it misses fall back to the existing value rather than going NULL. If there is no overlap, the column is added as a plain `enrich_<name>.<col>`.
|
|
544
556
|
|
|
545
557
|
**Example** — attach user cohort labels by `user_pseudo_id` (Dataform-declared table referenced by `{ schema, name }`):
|
|
546
558
|
|
|
@@ -548,7 +560,7 @@ For typical use cases this is the right tool; reach for `customSteps` only when
|
|
|
548
560
|
enrichments: [
|
|
549
561
|
{
|
|
550
562
|
name: 'cohorts',
|
|
551
|
-
level
|
|
563
|
+
// level omitted → defaults to 'row'
|
|
552
564
|
source: { schema: 'analytics', name: 'user_cohorts' },
|
|
553
565
|
joinKey: 'user_pseudo_id',
|
|
554
566
|
columns: ['cohort_label', 'lifecycle_stage'],
|
|
@@ -562,7 +574,7 @@ enrichments: [
|
|
|
562
574
|
enrichments: [
|
|
563
575
|
{
|
|
564
576
|
name: 'segments',
|
|
565
|
-
level: '
|
|
577
|
+
level: 'row',
|
|
566
578
|
source: '`my-project.analytics.daily_user_segments`',
|
|
567
579
|
joinKey: ['event_date', 'user_pseudo_id'],
|
|
568
580
|
columns: ['segment'],
|
|
@@ -579,7 +591,7 @@ enrichments: [
|
|
|
579
591
|
enrichments: [
|
|
580
592
|
{
|
|
581
593
|
name: 'titles',
|
|
582
|
-
level: '
|
|
594
|
+
level: 'row',
|
|
583
595
|
source: { schema: 'analytics', name: 'page_title_overrides' },
|
|
584
596
|
joinKey: 'page_location',
|
|
585
597
|
columns: ['page_title'], // overlaps the promoted column → coalesce(enrich.page_title, event_data.page_title)
|
|
@@ -588,9 +600,25 @@ enrichments: [
|
|
|
588
600
|
}
|
|
589
601
|
```
|
|
590
602
|
|
|
603
|
+
**Example** — item-level enrichment: attach product master data to each item via `item_id`. The enrichment flows into the `items` array struct; `margin_bucket` is added as a new item-struct field, and `item_category` overlap-coalesces against the original. Item-level enrichment columns do NOT appear at the row grain — they live inside `items[].<col>`:
|
|
604
|
+
|
|
605
|
+
```javascript
|
|
606
|
+
enrichments: [
|
|
607
|
+
{
|
|
608
|
+
name: 'products',
|
|
609
|
+
level: 'item',
|
|
610
|
+
source: { schema: 'analytics', name: 'product_master' },
|
|
611
|
+
joinKey: 'item_id', // joins on item.item_id
|
|
612
|
+
columns: ['margin_bucket', 'item_category'], // margin_bucket is additive; item_category overlap-coalesces
|
|
613
|
+
},
|
|
614
|
+
],
|
|
615
|
+
```
|
|
616
|
+
|
|
617
|
+
For `level: 'item'`, valid `joinKey` values are any field on the GA4 items struct (`item_id`, `item_category`, etc.) or any column on `event_data` (`user_pseudo_id`, `event_date`, etc.). A row-level and an item-level enrichment may share the same column name (e.g. both writing `cohort`) — the two columns target structurally distinct slots (`enhanced_events.cohort` at row grain vs `items[].cohort` inside the items array) and are not in collision.
|
|
618
|
+
|
|
591
619
|
> **Note:** Each enrichment generates a CTE named `enrich_<name>` at the top of the pipeline. The `enrich_*` namespace is part of the reserved-names contract — `customSteps` cannot use these names. The active reserved set includes only the names of enrichments actually configured.
|
|
592
620
|
|
|
593
|
-
> **Note:**
|
|
621
|
+
> **Note:** Row-level enrichment columns get auto-generated descriptions (`Added by enrichment '<name>' (joined on <joinKey> from <source>).` for new columns; `Coalesced by enrichment '<name>' (...; falls back to original on missed JOIN). Original: <description>` for overlapping columns). User-supplied `dataformTableConfig.columns` overrides win — the auto-generated description is the default. Item-level enrichment columns do not receive auto-generated descriptions (BigQuery does not surface per-field descriptions on STRUCT-array fields cleanly through Dataform's column-description mechanism).
|
|
594
622
|
|
|
595
623
|
> **Note:** `joinKey` and `columns` entries must be plain SQL identifiers — inline aliases like `'id as user_id'` are rejected at validation time. If your dim source uses a different column name, alias it in an upstream Dataform view and point `source` at that view.
|
|
596
624
|
|
package/documentation.js
CHANGED
|
@@ -173,8 +173,8 @@ const getColumnDescriptions = (config, columnMetadata) => {
|
|
|
173
173
|
// Item-level enrichments are not yet supported and throw at SQL gen time — skip here.
|
|
174
174
|
if (config && Array.isArray(config.enrichments) && config.enrichments.length > 0) {
|
|
175
175
|
config.enrichments.forEach(e => {
|
|
176
|
-
const level = e.level ?? '
|
|
177
|
-
if (level !== '
|
|
176
|
+
const level = e.level ?? 'row';
|
|
177
|
+
if (level !== 'row') return;
|
|
178
178
|
const joinKeys = Array.isArray(e.joinKey) ? e.joinKey : [e.joinKey];
|
|
179
179
|
const joinKeyText = joinKeys.join(', ');
|
|
180
180
|
const sourceText = renderEnrichmentSource(e.source);
|
package/helpers/ga4Transforms.js
CHANGED
|
@@ -97,7 +97,7 @@ const isFinalData = (detectionMethod, dayThreshold) => {
|
|
|
97
97
|
/**
|
|
98
98
|
* The standard GA4 BigQuery export top-level column names, based on the official schema.
|
|
99
99
|
*
|
|
100
|
-
* list updated 2026-
|
|
100
|
+
* list updated 2026-07-30
|
|
101
101
|
*/
|
|
102
102
|
const ga4ExportColumns = [
|
|
103
103
|
"event_date",
|
|
@@ -108,6 +108,7 @@ const ga4ExportColumns = [
|
|
|
108
108
|
"event_value_in_usd",
|
|
109
109
|
"event_bundle_sequence_id",
|
|
110
110
|
"event_server_timestamp_offset",
|
|
111
|
+
"event_original_occurrence_timestamp",
|
|
111
112
|
"user_id",
|
|
112
113
|
"user_pseudo_id",
|
|
113
114
|
"privacy_info",
|
|
@@ -140,6 +141,55 @@ const ga4ExportColumns = [
|
|
|
140
141
|
*/
|
|
141
142
|
const isGa4ExportColumn = (columnName) => ga4ExportColumns.includes(columnName);
|
|
142
143
|
|
|
144
|
+
/**
|
|
145
|
+
* The standard GA4 BigQuery export items-struct field names, based on the official schema.
|
|
146
|
+
* Listed in GA4's source order — `items_rebuilt`'s explicit struct construction emits fields
|
|
147
|
+
* in this order, and consumers may reasonably depend on the items-struct schema field order
|
|
148
|
+
* matching GA4's own.
|
|
149
|
+
*
|
|
150
|
+
* `item_params` is a nested REPEATED RECORD and projects through as a single struct entry
|
|
151
|
+
* (no per-key handling).
|
|
152
|
+
*
|
|
153
|
+
* list updated 2026-05-12
|
|
154
|
+
*/
|
|
155
|
+
const ga4ItemStructFields = [
|
|
156
|
+
"item_id",
|
|
157
|
+
"item_name",
|
|
158
|
+
"item_brand",
|
|
159
|
+
"item_variant",
|
|
160
|
+
"item_category",
|
|
161
|
+
"item_category2",
|
|
162
|
+
"item_category3",
|
|
163
|
+
"item_category4",
|
|
164
|
+
"item_category5",
|
|
165
|
+
"price_in_usd",
|
|
166
|
+
"price",
|
|
167
|
+
"quantity",
|
|
168
|
+
"item_revenue_in_usd",
|
|
169
|
+
"item_revenue",
|
|
170
|
+
"item_refund_in_usd",
|
|
171
|
+
"item_refund",
|
|
172
|
+
"coupon",
|
|
173
|
+
"affiliation",
|
|
174
|
+
"location_id",
|
|
175
|
+
"item_list_id",
|
|
176
|
+
"item_list_name",
|
|
177
|
+
"item_list_index",
|
|
178
|
+
"promotion_id",
|
|
179
|
+
"promotion_name",
|
|
180
|
+
"creative_name",
|
|
181
|
+
"creative_slot",
|
|
182
|
+
"item_params"
|
|
183
|
+
];
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Checks whether a given field name is part of the standard GA4 BigQuery export items struct.
|
|
187
|
+
*
|
|
188
|
+
* @param {string} fieldName - The name of the field to check.
|
|
189
|
+
* @returns {boolean} True if the field name is a standard items-struct field, otherwise false.
|
|
190
|
+
*/
|
|
191
|
+
const isGa4ItemStructField = (fieldName) => ga4ItemStructFields.includes(fieldName);
|
|
192
|
+
|
|
143
193
|
/**
|
|
144
194
|
* Generates a SQL CASE expression that determines the GA4 export type from a table suffix.
|
|
145
195
|
*
|
|
@@ -186,13 +236,17 @@ const itemListAttributionExpr = (lookbackType, timestampColumn, lookbackTimeMs)
|
|
|
186
236
|
frameBounds = `range between ${lookbackMicros} preceding and current row`;
|
|
187
237
|
}
|
|
188
238
|
|
|
189
|
-
|
|
239
|
+
// Suppress attribution for:
|
|
240
|
+
// - refund events (outside the selection-driven journey window)
|
|
241
|
+
// - unconsented events (user_pseudo_id is NULL) — attribution requires a visitor
|
|
242
|
+
// identity to stitch select_* events to later receivers within the same visitor.
|
|
243
|
+
return `if(event_name = 'refund' or user_pseudo_id is null, null, last_value(
|
|
190
244
|
if(${selectEvents}, ${structExpr}, null) ignore nulls
|
|
191
245
|
) over(
|
|
192
246
|
partition by ${partitionBy}
|
|
193
247
|
order by ${timestampColumn} asc
|
|
194
248
|
${frameBounds}
|
|
195
|
-
)`;
|
|
249
|
+
))`;
|
|
196
250
|
};
|
|
197
251
|
|
|
198
252
|
/**
|
|
@@ -209,6 +263,10 @@ const itemListAttributionExpr = (lookbackType, timestampColumn, lookbackTimeMs)
|
|
|
209
263
|
* the rows are interchangeable, so arbitrary row number assignment between them
|
|
210
264
|
* produces the same result.
|
|
211
265
|
*
|
|
266
|
+
* Unconsented events (user_pseudo_id is NULL) use an empty-string sentinel inside
|
|
267
|
+
* concat — without it, CONCAT NULL-propagates and the row_id becomes NULL, which
|
|
268
|
+
* would prevent enrichments from applying to such events.
|
|
269
|
+
*
|
|
212
270
|
* @param {string} ecommerceEventsFilter - Comma-separated, quoted list of event names
|
|
213
271
|
* (e.g., "'purchase', 'add_to_cart'").
|
|
214
272
|
* @returns {string} SQL expression that evaluates to the row id or NULL.
|
|
@@ -217,7 +275,7 @@ const itemRowId = (ecommerceEventsFilter) => {
|
|
|
217
275
|
return `if(
|
|
218
276
|
event_name in (${ecommerceEventsFilter}),
|
|
219
277
|
farm_fingerprint(concat(
|
|
220
|
-
user_pseudo_id,
|
|
278
|
+
ifnull(user_pseudo_id, ''),
|
|
221
279
|
cast(event_timestamp as string),
|
|
222
280
|
event_name,
|
|
223
281
|
to_json_string(items),
|
|
@@ -257,6 +315,8 @@ module.exports = {
|
|
|
257
315
|
isFinalData,
|
|
258
316
|
ga4ExportColumns,
|
|
259
317
|
isGa4ExportColumn,
|
|
318
|
+
ga4ItemStructFields,
|
|
319
|
+
isGa4ItemStructField,
|
|
260
320
|
getGa4ExportType,
|
|
261
321
|
itemListAttributionExpr,
|
|
262
322
|
itemRowId,
|
package/package.json
CHANGED
|
@@ -290,6 +290,7 @@
|
|
|
290
290
|
"event_value_in_usd": "Currency-converted value (in USD) of the event's 'value' parameter",
|
|
291
291
|
"event_bundle_sequence_id": "Sequential ID of the bundle in which the event was uploaded",
|
|
292
292
|
"event_server_timestamp_offset": "Timestamp offset between collection time and upload time in microseconds",
|
|
293
|
+
"event_original_occurrence_timestamp": "Timestamp in microseconds (UTC) of when the event originally occurred, for events that arrived at GA4 after the standard 72-hour collection window. Such events are processed in the current day's export (previously they were dropped); this field preserves the original event time. Null for events that arrived within 72 hours.",
|
|
293
294
|
"privacy_info": {
|
|
294
295
|
"description": "Consent status information for the user when consent mode is enabled",
|
|
295
296
|
"columns": {
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
"event_date": "Partition column. Always include in WHERE clauses to limit scanned data and reduce query cost",
|
|
3
3
|
"event_datetime": "Human-readable timestamp in the configured timezone. Use for time-of-day analysis and reporting",
|
|
4
4
|
"event_timestamp": "Microsecond-precision UTC timestamp. Use for precise event ordering and time difference calculations",
|
|
5
|
+
"event_original_occurrence_timestamp": "Use to identify events that arrived late (after 72 hours) and to recover their true occurrence time. On such rows event_timestamp reflects when GA4 processed the event, not when it actually happened. Filter WHERE event_original_occurrence_timestamp IS NOT NULL to isolate delayed events",
|
|
5
6
|
"event_name": "Primary event filter. Use in WHERE clauses to select specific event types (e.g. WHERE event_name = 'purchase')",
|
|
6
7
|
"session_id": "Use for counting unique sessions (COUNT(DISTINCT session_id)) and as a GROUP BY key for session-level aggregations",
|
|
7
8
|
"user_pseudo_id": "Device-level user identifier. Use for counting unique devices or as a fallback when merged_user_id is not needed",
|
|
@@ -69,8 +69,9 @@ const ga4EventsEnhancedConfig = {
|
|
|
69
69
|
// each entry is a queryBuilder step (raw {name, query} or structured {name, select, from, ...})
|
|
70
70
|
customSteps: [],
|
|
71
71
|
// declarative external-data enrichments joined into the pipeline
|
|
72
|
-
// each entry: { name,
|
|
73
|
-
//
|
|
72
|
+
// each entry: { name, source, joinKey, columns, level?, dedupe? }
|
|
73
|
+
// `level` is optional — defaults to 'row' (one row of the enclosing table per join match).
|
|
74
|
+
// 'item' targets the items[] array (GA4-specific, ecommerce events only).
|
|
74
75
|
enrichments: [],
|
|
75
76
|
};
|
|
76
77
|
|
|
@@ -162,8 +162,18 @@ const _generateEnhancedEventsSQL = (mergedConfig) => {
|
|
|
162
162
|
|
|
163
163
|
// item list attribution config
|
|
164
164
|
const itemListAttribution = mergedConfig.itemListAttribution;
|
|
165
|
-
|
|
166
|
-
|
|
165
|
+
|
|
166
|
+
// Build enrichment-source CTEs and gather per-level join/column data. The utility routes
|
|
167
|
+
// row-level and item-level entries through separate output channels. Done up here so the
|
|
168
|
+
// items-scaffold activation state is known before building event_data (which needs
|
|
169
|
+
// _item_row_id when the scaffold is active for any reason).
|
|
170
|
+
const { steps: enrichmentSteps, row: rowEnrichments, item: itemEnrichments }
|
|
171
|
+
= utils.buildEnrichments(mergedConfig.enrichments);
|
|
172
|
+
const itemEnrichmentsActive = itemEnrichments.joins.length > 0;
|
|
173
|
+
const itemsScaffoldActive = !!itemListAttribution || itemEnrichmentsActive;
|
|
174
|
+
|
|
175
|
+
const ecommerceEventsFilter = itemsScaffoldActive
|
|
176
|
+
? helpers.ga4EcommerceEvents.map(e => `'${e}'`).join(', ')
|
|
167
177
|
: null;
|
|
168
178
|
|
|
169
179
|
// auto-adjust bufferDays for time-based item list attribution lookback
|
|
@@ -220,7 +230,7 @@ const _generateEnhancedEventsSQL = (mergedConfig) => {
|
|
|
220
230
|
// ecommerce
|
|
221
231
|
ecommerce: helpers.fixEcommerceStruct('ecommerce'),
|
|
222
232
|
// assign a unique row id, used for handling item-level attribution and enrichment
|
|
223
|
-
_item_row_id:
|
|
233
|
+
_item_row_id: itemsScaffoldActive ? helpers.itemRowId(ecommerceEventsFilter) : undefined,
|
|
224
234
|
// flag if the data is "final" and is not expected to change anymore
|
|
225
235
|
data_is_final: helpers.isFinalData(mergedConfig.dataIsFinal.detectionMethod, mergedConfig.dataIsFinal.dayThreshold),
|
|
226
236
|
export_type: helpers.getGa4ExportType('_table_suffix'),
|
|
@@ -263,76 +273,164 @@ ${excludedEventsSQL}`,
|
|
|
263
273
|
'group by': 'session_id',
|
|
264
274
|
};
|
|
265
275
|
|
|
276
|
+
// Validate item-level joinKey columns and collect any event_data columns that need to
|
|
277
|
+
// be carried up to items_unnested as top-level columns (so the LEFT JOIN inside
|
|
278
|
+
// items_rebuilt can USING(...) on them). Item-struct fields are already top-level on
|
|
279
|
+
// items_unnested and need no extension.
|
|
280
|
+
const itemJoinKeysFromEventData = new Set();
|
|
281
|
+
for (const [i, e] of (mergedConfig.enrichments ?? []).entries()) {
|
|
282
|
+
const level = e.level ?? 'row';
|
|
283
|
+
if (level !== 'item') continue;
|
|
284
|
+
const joinKeys = Array.isArray(e.joinKey) ? e.joinKey : [e.joinKey];
|
|
285
|
+
for (const c of joinKeys) {
|
|
286
|
+
if (helpers.ga4ItemStructFields.includes(c)) {
|
|
287
|
+
// Already a top-level column on items_unnested.
|
|
288
|
+
} else if (c in eventDataStep.select.columns && eventDataStep.select.columns[c] !== undefined) {
|
|
289
|
+
itemJoinKeysFromEventData.add(c);
|
|
290
|
+
} else {
|
|
291
|
+
throw new Error(
|
|
292
|
+
`config.enrichments[${i}] (name: '${e.name}') uses item-level joinKey '${c}', ` +
|
|
293
|
+
`which is neither a field on the GA4 items struct (helpers.ga4ItemStructFields) ` +
|
|
294
|
+
`nor a column on event_data. Valid item-level joinKeys are item-struct fields ` +
|
|
295
|
+
`(e.g. item_id, item_category) or any event_data column (e.g. user_pseudo_id, event_date).`
|
|
296
|
+
);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
266
301
|
// Shared item-array CTEs:
|
|
267
|
-
// 1. items_unnested: unnest items from ecommerce events
|
|
268
|
-
//
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
);
|
|
302
|
+
// 1. items_unnested: unnest items from ecommerce events; LAST_VALUE attribution window
|
|
303
|
+
// is emitted only when itemListAttribution is configured.
|
|
304
|
+
// 2. items_rebuilt: re-aggregate items via explicit struct(...) construction;
|
|
305
|
+
// LEFT JOIN enrich_<name> for each item-level enrichment.
|
|
306
|
+
// Activation: emitted when EITHER itemListAttribution is configured OR at least one
|
|
307
|
+
// item-level enrichment is present.
|
|
308
|
+
const itemListSteps = itemsScaffoldActive ? (() => {
|
|
275
309
|
const passthroughEvents = `event_name in ('view_item_list', 'select_item', 'view_promotion', 'select_promotion')`;
|
|
276
310
|
|
|
311
|
+
// Flatten the item struct: every standard items-struct field is selected as a
|
|
312
|
+
// top-level column of items_unnested. This makes downstream joins simpler
|
|
313
|
+
// (LEFT JOIN ... USING(item_id) works without aliasing tricks) and lets items_rebuilt
|
|
314
|
+
// reference fields as bare column names instead of `item.<col>`.
|
|
315
|
+
const itemFieldColumns = {};
|
|
316
|
+
for (const f of helpers.ga4ItemStructFields) {
|
|
317
|
+
itemFieldColumns[f] = `item.${f}`;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// Carry up any event_data joinKey columns used by item-level enrichments so the
|
|
321
|
+
// USING(...) clause in items_rebuilt can bind against top-level identifiers.
|
|
322
|
+
// Skip ones already in the base columns above
|
|
323
|
+
const baseColumnNames = new Set(['_item_row_id', 'event_name', ...Object.keys(itemFieldColumns)]);
|
|
324
|
+
const extraJoinKeyColumns = {};
|
|
325
|
+
for (const c of itemJoinKeysFromEventData) {
|
|
326
|
+
if (!baseColumnNames.has(c)) {
|
|
327
|
+
extraJoinKeyColumns[c] = c;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// items_unnested base columns. The _item_list_attr struct (LAST_VALUE window) is
|
|
332
|
+
// added only when itemListAttribution is configured — when only item enrichments
|
|
333
|
+
// are active, the window function is omitted entirely for cleaner SQL.
|
|
334
|
+
const unnestedSelectColumns = {
|
|
335
|
+
'_item_row_id': '_item_row_id',
|
|
336
|
+
'event_name': 'event_name',
|
|
337
|
+
...itemFieldColumns,
|
|
338
|
+
...extraJoinKeyColumns,
|
|
339
|
+
};
|
|
340
|
+
if (itemListAttribution) {
|
|
341
|
+
unnestedSelectColumns._item_list_attr = helpers.itemListAttributionExpr(
|
|
342
|
+
itemListAttribution.lookbackType,
|
|
343
|
+
timestampColumn,
|
|
344
|
+
itemListAttribution.lookbackTimeMs
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
|
|
277
348
|
const unnestedStep = {
|
|
278
349
|
name: 'items_unnested',
|
|
279
|
-
select: {
|
|
280
|
-
columns: {
|
|
281
|
-
'_item_row_id': '_item_row_id',
|
|
282
|
-
'event_name': 'event_name',
|
|
283
|
-
// event_date is carried forward for ability to use it in data enrichment joins
|
|
284
|
-
'event_date': 'event_date',
|
|
285
|
-
'item': 'item',
|
|
286
|
-
'_item_list_attr': attrExpr,
|
|
287
|
-
},
|
|
288
|
-
},
|
|
350
|
+
select: { columns: unnestedSelectColumns },
|
|
289
351
|
from: 'event_data, unnest(items) as item',
|
|
290
352
|
where: `event_name in (${ecommerceEventsFilter})`,
|
|
291
353
|
};
|
|
292
354
|
|
|
355
|
+
// Build the per-field expression map for the items struct. Seed with the canonical
|
|
356
|
+
// GA4 items-struct fields — each references the matching top-level column on
|
|
357
|
+
// items_unnested. When itemListAttribution is configured, override the three
|
|
358
|
+
// attribution entries with their package-generated coalesce-with-passthrough
|
|
359
|
+
// expressions. Item-level enrichment columns layer on top via the spread below.
|
|
360
|
+
// References are qualified with `items_unnested.` so that overlapping item-level
|
|
361
|
+
// enrichments (which JOIN against enrich_<name> CTEs that may share column names)
|
|
362
|
+
// do not produce ambiguous bare-column references.
|
|
363
|
+
const preItemExpressions = {};
|
|
364
|
+
for (const f of helpers.ga4ItemStructFields) {
|
|
365
|
+
preItemExpressions[f] = `items_unnested.${f}`;
|
|
366
|
+
}
|
|
367
|
+
if (itemListAttribution) {
|
|
368
|
+
preItemExpressions.item_list_name = `coalesce(if(${passthroughEvents}, items_unnested.item_list_name, _item_list_attr.item_list_name), '(not set)')`;
|
|
369
|
+
preItemExpressions.item_list_id = `coalesce(if(${passthroughEvents}, items_unnested.item_list_id, _item_list_attr.item_list_id), '(not set)')`;
|
|
370
|
+
preItemExpressions.item_list_index = `coalesce(if(${passthroughEvents}, items_unnested.item_list_index, _item_list_attr.item_list_index))`;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// Wrap overlapping item-level enrichment columns in coalesce(<enrichExpr>, <originalExpr>)
|
|
374
|
+
// so a missed JOIN falls back to the existing item field value. Purely additive
|
|
375
|
+
// columns (no overlap) pass through unchanged.
|
|
376
|
+
const wrappedItemEnrichmentColumns = {};
|
|
377
|
+
for (const [col, enrichExpr] of Object.entries(itemEnrichments.columns)) {
|
|
378
|
+
const originalExpr = preItemExpressions[col];
|
|
379
|
+
wrappedItemEnrichmentColumns[col] = originalExpr
|
|
380
|
+
? `coalesce(${enrichExpr}, ${originalExpr})`
|
|
381
|
+
: enrichExpr;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// Final struct: standard fields first, then enrichment overrides spread on top
|
|
385
|
+
// (overlapping keys replace preItemExpressions entries; additive keys are appended).
|
|
386
|
+
const finalItemStructFields = { ...preItemExpressions, ...wrappedItemEnrichmentColumns };
|
|
387
|
+
|
|
388
|
+
const itemStructClauses = Object.entries(finalItemStructFields)
|
|
389
|
+
.map(([col, expr]) => `${expr} as ${col}`)
|
|
390
|
+
.join(',\n ');
|
|
391
|
+
|
|
293
392
|
const rebuiltStep = {
|
|
294
393
|
name: 'items_rebuilt',
|
|
295
394
|
select: {
|
|
296
395
|
columns: {
|
|
297
396
|
'_item_row_id': '_item_row_id',
|
|
298
|
-
'items': `array_agg(
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
coalesce(if(${passthroughEvents}, item.item_list_id, _item_list_attr.item_list_id), '(not set)') as item_list_id,
|
|
302
|
-
coalesce(if(${passthroughEvents}, item.item_list_index, _item_list_attr.item_list_index)) as item_list_index
|
|
303
|
-
))
|
|
304
|
-
)`,
|
|
397
|
+
'items': `array_agg(struct(
|
|
398
|
+
${itemStructClauses}
|
|
399
|
+
))`,
|
|
305
400
|
},
|
|
306
401
|
},
|
|
307
402
|
from: 'items_unnested',
|
|
308
403
|
'group by': '_item_row_id',
|
|
309
404
|
};
|
|
405
|
+
// Item-level enrichment joins (only attach when present). Each enrichment's LEFT JOIN
|
|
406
|
+
// binds against top-level columns on items_unnested (item-struct fields, or event_data
|
|
407
|
+
// joinKey columns carried up via extraJoinKeyColumns above).
|
|
408
|
+
if (itemEnrichmentsActive) {
|
|
409
|
+
rebuiltStep.joins = itemEnrichments.joins;
|
|
410
|
+
}
|
|
310
411
|
|
|
311
412
|
return [unnestedStep, rebuiltStep];
|
|
312
413
|
})() : null;
|
|
313
414
|
|
|
314
415
|
const finalColumnOrder = getFinalColumnOrder(eventDataStep, sessionDataStep);
|
|
315
416
|
|
|
316
|
-
// When
|
|
317
|
-
//
|
|
417
|
+
// When the items scaffold is active, override the items column and exclude _item_row_id.
|
|
418
|
+
// ifnull(..., []) preserves the empty-array shape for events that have no items_rebuilt
|
|
419
|
+
// match (non-ecommerce events, or ecommerce events with empty items arrays). The empty
|
|
420
|
+
// array literal is type-inferred from items_rebuilt.items, which includes any item-level
|
|
421
|
+
// enrichment columns — so additive enrichments don't cause a struct-schema mismatch.
|
|
318
422
|
const itemListOverrides = itemListSteps ? {
|
|
319
|
-
items: '
|
|
423
|
+
items: 'ifnull(items_rebuilt.items, [])',
|
|
320
424
|
} : {};
|
|
321
425
|
const itemListExcludedColumns = itemListSteps ? ['_item_row_id'] : [];
|
|
322
426
|
|
|
323
|
-
//
|
|
324
|
-
//
|
|
325
|
-
const { steps: enrichmentSteps, joins: enrichmentJoins, columns: enrichmentColumns,
|
|
326
|
-
columnNames: enrichmentColumnNames } = utils.buildEnrichments(mergedConfig.enrichments);
|
|
327
|
-
|
|
328
|
-
// Wrap overlapping enrichment columns in coalesce(enrich_<name>.<col>, <original>) so a
|
|
329
|
-
// missed JOIN falls back to the existing value. Purely additive columns (no overlap)
|
|
427
|
+
// Wrap overlapping row-level enrichment columns in coalesce(enrich_<name>.<col>, <original>)
|
|
428
|
+
// so a missed JOIN falls back to the existing value. Purely additive columns (no overlap)
|
|
330
429
|
// pass through unchanged. Source-of-original precedence matches the final SELECT's spread
|
|
331
430
|
// order: itemListOverrides first (overrides finalColumnOrder for `items`), then
|
|
332
431
|
// session_data (wins over event_data in getFinalColumnOrder when both have the column).
|
|
333
|
-
|
|
334
|
-
const
|
|
335
|
-
for (const [col, enrichExpr] of Object.entries(enrichmentColumns)) {
|
|
432
|
+
const wrappedRowEnrichmentColumns = {};
|
|
433
|
+
for (const [col, enrichExpr] of Object.entries(rowEnrichments.columns)) {
|
|
336
434
|
let originalExpr;
|
|
337
435
|
if (col in itemListOverrides) {
|
|
338
436
|
originalExpr = itemListOverrides[col];
|
|
@@ -341,7 +439,7 @@ ${excludedEventsSQL}`,
|
|
|
341
439
|
} else if (col in eventDataStep.select.columns && eventDataStep.select.columns[col] !== undefined) {
|
|
342
440
|
originalExpr = `event_data.${col}`;
|
|
343
441
|
}
|
|
344
|
-
|
|
442
|
+
wrappedRowEnrichmentColumns[col] = originalExpr
|
|
345
443
|
? `coalesce(${enrichExpr}, ${originalExpr})`
|
|
346
444
|
: enrichExpr;
|
|
347
445
|
}
|
|
@@ -351,7 +449,7 @@ ${excludedEventsSQL}`,
|
|
|
351
449
|
const alreadyMapped = [
|
|
352
450
|
...Object.keys(finalColumnOrder),
|
|
353
451
|
...Object.keys(itemListOverrides),
|
|
354
|
-
...
|
|
452
|
+
...rowEnrichments.columnNames,
|
|
355
453
|
'entrances',
|
|
356
454
|
mergedConfig.sessionParams.length > 0 ? 'session_params_prep' : undefined,
|
|
357
455
|
'data_is_final',
|
|
@@ -368,8 +466,8 @@ ${excludedEventsSQL}`,
|
|
|
368
466
|
// get the most important columns in the correct order
|
|
369
467
|
...finalColumnOrder,
|
|
370
468
|
...itemListOverrides,
|
|
371
|
-
//
|
|
372
|
-
...
|
|
469
|
+
// row-level enrichment columns: coalesce with the original when overlapping; otherwise add.
|
|
470
|
+
...wrappedRowEnrichmentColumns,
|
|
373
471
|
// explicit pass-throughs for the rest of event_data and session_data
|
|
374
472
|
...utils.buildQualifiedPassThroughs(eventDataStep, alreadyMapped),
|
|
375
473
|
...utils.buildQualifiedPassThroughs(sessionDataStep, alreadyMapped),
|
|
@@ -391,8 +489,8 @@ ${excludedEventsSQL}`,
|
|
|
391
489
|
table: 'session_data',
|
|
392
490
|
on: 'using(session_id)'
|
|
393
491
|
},
|
|
394
|
-
// The left joins for the enrichment ctes
|
|
395
|
-
...
|
|
492
|
+
// The left joins for the row-level enrichment ctes
|
|
493
|
+
...rowEnrichments.joins,
|
|
396
494
|
],
|
|
397
495
|
where: helpers.incrementalDateFilter(mergedConfig)
|
|
398
496
|
};
|
|
@@ -201,11 +201,11 @@ const validateEnhancedEventsConfig = (config, options = {}) => {
|
|
|
201
201
|
}
|
|
202
202
|
}
|
|
203
203
|
|
|
204
|
-
// customSteps - optional array of queryBuilder step objects appended to the pipeline
|
|
205
|
-
//
|
|
204
|
+
// customSteps - optional array of queryBuilder step objects appended to the pipeline.
|
|
205
|
+
// Config-shape checks only: array, objects with non-empty name, no duplicates within customSteps.
|
|
206
206
|
// Step-shape validation (clause keys, etc.) deferred to queryBuilder.
|
|
207
|
-
// Collision-with-package-names check deferred to _generateEnhancedEventsSQL
|
|
208
|
-
//
|
|
207
|
+
// Collision-with-package-names check deferred to _generateEnhancedEventsSQL, since the
|
|
208
|
+
// reserved set is config-dependent (e.g. item_list_* only exist when itemListAttribution is on).
|
|
209
209
|
if (config.customSteps !== undefined) {
|
|
210
210
|
if (!Array.isArray(config.customSteps)) {
|
|
211
211
|
throw new Error(`config.customSteps must be an array. Received: ${JSON.stringify(config.customSteps)}`);
|
|
@@ -227,14 +227,14 @@ const validateEnhancedEventsConfig = (config, options = {}) => {
|
|
|
227
227
|
}
|
|
228
228
|
|
|
229
229
|
// enrichments - optional array of declarative external-data enrichment specs.
|
|
230
|
-
//
|
|
231
|
-
//
|
|
232
|
-
//
|
|
230
|
+
// Config-shape checks only. Reserved-name collision and item-level joinKey resolution
|
|
231
|
+
// happen in _generateEnhancedEventsSQL, where the reserved set and item-level join targets
|
|
232
|
+
// are derived from the resolved config.
|
|
233
233
|
if (config.enrichments !== undefined) {
|
|
234
234
|
if (!Array.isArray(config.enrichments)) {
|
|
235
235
|
throw new Error(`config.enrichments must be an array. Received: ${JSON.stringify(config.enrichments)}`);
|
|
236
236
|
}
|
|
237
|
-
const validLevels = ['
|
|
237
|
+
const validLevels = ['row', 'item'];
|
|
238
238
|
const seenNames = new Set();
|
|
239
239
|
for (let i = 0; i < config.enrichments.length; i++) {
|
|
240
240
|
const entry = config.enrichments[i];
|
package/utils.js
CHANGED
|
@@ -515,48 +515,53 @@ const buildPassThroughs = (explicitColumns, sourceColumns) => {
|
|
|
515
515
|
|
|
516
516
|
/**
|
|
517
517
|
* Builds the per-enrichment CTE definitions, JOIN clauses, and column-name mappings for the
|
|
518
|
-
* declarative `enrichments` feature.
|
|
518
|
+
* declarative `enrichments` feature. Routes row-level and item-level entries through
|
|
519
|
+
* separate output channels so the caller can attach them to different downstream CTEs.
|
|
519
520
|
*
|
|
520
521
|
* Pure config-to-data mapping. No knowledge of downstream CTEs or specific table modules —
|
|
521
522
|
* intended to be called by any table module that exposes an `enrichments` config field.
|
|
522
523
|
*
|
|
523
|
-
* Encapsulates
|
|
524
|
-
* - level
|
|
525
|
-
*
|
|
524
|
+
* Encapsulates one generation-time throw:
|
|
525
|
+
* - Same-level enrichment-vs-enrichment column collisions (two row-level enrichments or
|
|
526
|
+
* two item-level enrichments targeting the same column). Cross-level same-name is allowed —
|
|
527
|
+
* the two columns target structurally distinct slots (e.g. `enhanced_events.<col>` vs
|
|
528
|
+
* `items[].<col>`).
|
|
526
529
|
*
|
|
527
530
|
* @param {Array<Object>} enrichments - Validated enrichment entries. Each entry has fields:
|
|
528
|
-
* { name,
|
|
529
|
-
*
|
|
530
|
-
*
|
|
531
|
-
*
|
|
532
|
-
* - `
|
|
533
|
-
*
|
|
534
|
-
* - `
|
|
535
|
-
*
|
|
536
|
-
*
|
|
537
|
-
*
|
|
531
|
+
* { name, source, joinKey, columns, level?, dedupe? }. `level` is 'row' (default) or 'item'.
|
|
532
|
+
* 'row' means one row of the enclosing table per join match; 'item' targets a nested array
|
|
533
|
+
* (currently only the GA4 items[] array).
|
|
534
|
+
* @returns {Object} A struct with four fields:
|
|
535
|
+
* - `steps` — array of queryBuilder source-CTE step definitions (one `enrich_<name>` per
|
|
536
|
+
* entry, regardless of level — all source CTEs go to the top of the pipeline).
|
|
537
|
+
* - `row` — { joins, columns, columnNames } for row-level enrichments. Caller attaches
|
|
538
|
+
* `joins` to the row-grained downstream CTE (e.g. `enhanced_events`) and spreads `columns`
|
|
539
|
+
* into that CTE's `select.columns`.
|
|
540
|
+
* - `item` — { joins, columns, columnNames } for item-level enrichments. Caller attaches
|
|
541
|
+
* `joins` to the item-grained downstream CTE (e.g. `items_rebuilt`) and folds `columns`
|
|
542
|
+
* into that CTE's struct construction.
|
|
543
|
+
* - `columnOwner` — map of `{ <column>: { i, name, level } }` recording which enrichment
|
|
544
|
+
* owns each column. The `level` field distinguishes cross-level same-name entries.
|
|
538
545
|
*
|
|
539
|
-
* @throws {Error} If
|
|
540
|
-
*
|
|
546
|
+
* @throws {Error} If two same-level enrichments target the same column name (with both
|
|
547
|
+
* enrichment names and the conflicting column in the error message).
|
|
541
548
|
*
|
|
542
549
|
* @example
|
|
543
|
-
* const { steps,
|
|
550
|
+
* const { steps, row, item } = buildEnrichments(config.enrichments);
|
|
551
|
+
* // row.joins → attach to enhanced_events; row.columns → spread into enhanced_events
|
|
552
|
+
* // item.joins → attach to items_rebuilt; item.columns → fold into items struct
|
|
544
553
|
*/
|
|
545
554
|
const buildEnrichments = (enrichments) => {
|
|
546
555
|
const steps = [];
|
|
547
|
-
const
|
|
548
|
-
|
|
549
|
-
|
|
556
|
+
const channels = {
|
|
557
|
+
row: { joins: [], columns: {}, columnNames: new Set() },
|
|
558
|
+
item: { joins: [], columns: {}, columnNames: new Set() },
|
|
559
|
+
};
|
|
550
560
|
const columnOwner = {};
|
|
551
561
|
|
|
552
562
|
for (const [i, e] of (enrichments ?? []).entries()) {
|
|
553
|
-
const level = e.level ?? '
|
|
554
|
-
|
|
555
|
-
throw new Error(
|
|
556
|
-
`config.enrichments[${i}] uses level: 'item', which is not yet supported in this version. ` +
|
|
557
|
-
`Item-level enrichments will ship in a future release; see design_docs/planned/data-enrichments.md.`
|
|
558
|
-
);
|
|
559
|
-
}
|
|
563
|
+
const level = e.level ?? 'row';
|
|
564
|
+
const channel = channels[level];
|
|
560
565
|
const joinKeys = Array.isArray(e.joinKey) ? e.joinKey : [e.joinKey];
|
|
561
566
|
const cteName = `enrich_${e.name}`;
|
|
562
567
|
|
|
@@ -573,24 +578,29 @@ const buildEnrichments = (enrichments) => {
|
|
|
573
578
|
}
|
|
574
579
|
steps.push(sourceStep);
|
|
575
580
|
|
|
576
|
-
joins.push({ type: 'left', table: cteName, on: `using(${joinKeys.join(', ')})` });
|
|
581
|
+
channel.joins.push({ type: 'left', table: cteName, on: `using(${joinKeys.join(', ')})` });
|
|
577
582
|
|
|
578
583
|
for (const c of e.columns) {
|
|
579
|
-
|
|
584
|
+
// Same-level collision throw. Cross-level same-name is allowed because the two
|
|
585
|
+
// columns target structurally distinct output slots (event_data vs items[]).
|
|
586
|
+
if (channel.columnNames.has(c)) {
|
|
580
587
|
const owner = columnOwner[c];
|
|
581
588
|
throw new Error(
|
|
582
589
|
`config.enrichments[${i}] (name: '${e.name}') and config.enrichments[${owner.i}] ` +
|
|
583
|
-
`(name: '${owner.name}') both target column '${c}'. ` +
|
|
584
|
-
`Two enrichments cannot write the same column; rename one in source SQL or pick a different name.`
|
|
590
|
+
`(name: '${owner.name}') both target column '${c}' at level '${level}'. ` +
|
|
591
|
+
`Two enrichments cannot write the same column at the same level; rename one in source SQL or pick a different name.`
|
|
585
592
|
);
|
|
586
593
|
}
|
|
587
|
-
columns[c] = `${cteName}.${c}`;
|
|
588
|
-
columnNames.add(c);
|
|
589
|
-
columnOwner
|
|
594
|
+
channel.columns[c] = `${cteName}.${c}`;
|
|
595
|
+
channel.columnNames.add(c);
|
|
596
|
+
// columnOwner is keyed by column name; if the same name appears at different
|
|
597
|
+
// levels, the second-writer entry wins, but we record level so diagnostics
|
|
598
|
+
// distinguish them. Same-level collisions throw above before reaching here.
|
|
599
|
+
columnOwner[c] = { i, name: e.name, level };
|
|
590
600
|
}
|
|
591
601
|
}
|
|
592
602
|
|
|
593
|
-
return { steps,
|
|
603
|
+
return { steps, row: channels.row, item: channels.item, columnOwner };
|
|
594
604
|
};
|
|
595
605
|
|
|
596
606
|
|