ga4-export-fixer 0.3.2-dev.0 → 0.3.2-dev.2

package/columns/tableAgentInstructions.json

@@ -0,0 +1,130 @@
+{
+    "synonyms": [
+        {
+            "terms": ["users", "unique users", "visitors", "unique visitors"],
+            "sql": "COUNT(DISTINCT user_pseudo_id) or COUNT(DISTINCT merged_user_id) for cross-device",
+            "dependsOn": ["user_pseudo_id"]
+        },
+        {
+            "terms": ["sessions", "visits", "unique sessions"],
+            "sql": "COUNT(DISTINCT session_id)",
+            "dependsOn": ["session_id"]
+        },
+        {
+            "terms": ["page views", "pageviews", "page view count"],
+            "sql": "COUNT(*) WHERE event_name = 'page_view'",
+            "dependsOn": ["event_name"]
+        },
+        {
+            "terms": ["transactions", "purchases", "orders"],
+            "sql": "COUNT(*) WHERE event_name = 'purchase'",
+            "dependsOn": ["ecommerce"]
+        },
+        {
+            "terms": ["revenue", "sales", "purchase revenue", "total revenue"],
+            "sql": "SUM(ecommerce.purchase_revenue) WHERE event_name = 'purchase'",
+            "dependsOn": ["ecommerce"]
+        },
+        {
+            "terms": ["page URL", "page location", "URL", "current page"],
+            "sql": "page_location (full URL) or page.hostname, page.path, page.query (parsed components)",
+            "dependsOn": ["page_location"]
+        },
+        {
+            "terms": ["landing page", "entry page", "first page"],
+            "sql": "landing_page.hostname, landing_page.path",
+            "dependsOn": ["landing_page"]
+        },
+        {
+            "terms": ["traffic source", "channel", "medium", "campaign", "source", "UTM"],
+            "sql": "collected_traffic_source (event-level), session_first_traffic_source (session first-touch), session_traffic_source_last_click (session last-click attribution)",
+            "dependsOn": ["collected_traffic_source", "session_first_traffic_source"]
+        },
+        {
+            "terms": ["device type", "device category", "device", "mobile vs desktop"],
+            "sql": "device.category (values: mobile, tablet, desktop)",
+            "dependsOn": ["device"]
+        },
+        {
+            "terms": ["country", "location", "region", "city", "geography", "geo"],
+            "sql": "geo.country, geo.region, geo.city",
+            "dependsOn": ["geo"]
+        }
+    ],
+    "keyFields": [
+        {
+            "field": "event_date",
+            "note": "Partition column. Always include in WHERE to limit scanned data and cost.",
+            "dependsOn": []
+        },
+        {
+            "field": "event_name",
+            "note": "Primary event type filter (e.g. 'page_view', 'purchase', 'scroll').",
+            "dependsOn": []
+        },
+        {
+            "field": "user_pseudo_id",
+            "note": "Device-level user identifier. COUNT(DISTINCT) for unique users.",
+            "dependsOn": []
+        },
+        {
+            "field": "session_id",
+            "note": "Unique session identifier. COUNT(DISTINCT) for sessions, GROUP BY for session metrics.",
+            "dependsOn": ["session_id"]
+        },
+        {
+            "field": "merged_user_id",
+            "note": "Cross-device user identifier. Prefer over user_pseudo_id for user-level analysis.",
+            "dependsOn": []
+        },
+        {
+            "field": "page_location",
+            "note": "Full page URL. Use page.hostname and page.path for structured analysis.",
+            "dependsOn": ["page_location"]
+        },
+        {
+            "field": "ecommerce.purchase_revenue",
+            "note": "Revenue per transaction. Filter to event_name = 'purchase' before aggregating.",
+            "dependsOn": ["ecommerce"]
+        },
+        {
+            "field": "session_first_traffic_source",
+            "note": "Session acquisition source. Use .manual_source and .manual_medium for UTM data.",
+            "dependsOn": ["session_first_traffic_source"]
+        },
+        {
+            "field": "device.category",
+            "note": "Device type segmentation (mobile, tablet, desktop).",
+            "dependsOn": ["device"]
+        },
+        {
+            "field": "geo.country",
+            "note": "Geographic segmentation by country.",
+            "dependsOn": ["geo"]
+        },
+        {
+            "field": "event_datetime",
+            "note": "Event timestamp in the configured timezone. Use for time-of-day analysis.",
+            "dependsOn": []
+        },
+        {
+            "field": "data_is_final",
+            "note": "Data stability flag. WHERE data_is_final = true for stable data only.",
+            "dependsOn": []
+        }
+    ],
+    "filteringGuidance": [
+        {"text": "Always filter on event_date (partition column) to reduce query cost.", "dependsOn": []},
+        {"text": "Use event_name to select specific event types (e.g. WHERE event_name = 'page_view').", "dependsOn": []},
+        {"text": "For user counts: COUNT(DISTINCT user_pseudo_id) for device-level, COUNT(DISTINCT merged_user_id) for cross-device.", "dependsOn": []},
+        {"text": "For session metrics: GROUP BY session_id.", "dependsOn": ["session_id"]},
+        {"text": "For ecommerce revenue: filter WHERE event_name = 'purchase' before aggregating ecommerce.purchase_revenue.", "dependsOn": ["ecommerce"]},
+        {"text": "For traffic source analysis: use session_first_traffic_source for first-touch, session_traffic_source_last_click for last-click attribution.", "dependsOn": ["session_first_traffic_source"]},
+        {"text": "Nested arrays (event_params, items): use CROSS JOIN UNNEST(...) to access individual values.", "dependsOn": []},
+        {"text": "Use data_is_final = true to exclude data that may still change in future refreshes.", "dependsOn": []}
+    ],
+    "eventVocabulary": {
+        "autoCollectedAndEnhanced": ["page_view", "session_start", "first_visit", "user_engagement", "scroll", "click", "file_download", "video_start", "video_progress", "video_complete", "view_search_results", "form_start", "form_submit"],
+        "ecommerce": ["purchase", "add_to_cart", "begin_checkout", "view_item", "view_item_list", "add_to_wishlist", "remove_from_cart", "add_payment_info", "add_shipping_info", "refund"]
+    }
+}

package/documentation.js

@@ -1,6 +1,8 @@
 const columnDescriptions = require('./columns/columnDescriptions.json');
 const columnLineage = require('./columns/columnLineage.json');
 const columnTypicalUse = require('./columns/columnTypicalUse.json');
+const tableAgentInstructions = require('./columns/tableAgentInstructions.json');
+const constants = require('./constants');
 
 /**
  * Composes a multi-section column description string from individual sections.
@@ -168,9 +170,134 @@ const getColumnDescriptions = (config) => {
     return descriptions;
 };
 
+/**
+ * Checks whether a column (or its parent struct) is excluded by the config.
+ *
+ * @param {string[]} dependsOn - Column names this entry depends on.
+ * @param {string[]} excludedColumns - Combined excluded columns from config.
+ * @returns {boolean} True if ALL dependsOn columns are excluded.
+ */
+const isExcluded = (dependsOn, excludedColumns) => {
+    if (!dependsOn || dependsOn.length === 0) return false;
+    return dependsOn.every(col => excludedColumns.includes(col));
+};
+
+/**
+ * Composes the full table description for ga4_events_enhanced, including
+ * AI agent instructions (key fields, synonyms, filtering guidance, event vocabulary)
+ * and the existing table features and config JSON dump.
+ *
+ * @param {Object} config - The merged configuration object.
+ * @returns {string} The composed table description.
+ */
+const getTableDescription = (config) => {
+    // Only use user-configured excludedColumns for filtering AI instructions.
+    // defaultExcludedColumns refers to raw GA4 export columns excluded during extraction
+    // (e.g. session_id is excluded from the raw export but exists as a derived column in the final table).
+    const excludedColumns = config.excludedColumns || [];
+
+    const excludedEvents = [
+        ...(config.defaultExcludedEvents || []),
+        ...(config.excludedEvents || []),
+    ];
+
+    const sections = [];
+
+    // 1. Overview
+    const overviewLines = [
+        'GA4 Events Enhanced',
+        '',
+        'An enhanced version of the GA4 BigQuery export. Each row is one event.',
+    ];
+    if (config.timezone) {
+        overviewLines.push(`Timezone: ${config.timezone}.`);
+    }
+    sections.push(overviewLines.join('\n'));
+
+    // 2. Key Fields
+    const keyFieldLines = tableAgentInstructions.keyFields
+        .filter(kf => !isExcluded(kf.dependsOn, excludedColumns))
+        .map(kf => `- ${kf.field}: ${kf.note}`);
+
+    // Add promoted event params
+    if (config.eventParamsToColumns && config.eventParamsToColumns.length > 0) {
+        config.eventParamsToColumns.forEach(p => {
+            const columnName = p.columnName || p.name;
+            keyFieldLines.push(`- ${columnName}: Promoted event parameter '${p.name}'. Available as a top-level column for direct filtering.`);
+        });
+    }
+
+    if (keyFieldLines.length > 0) {
+        sections.push('KEY FIELDS:\n' + keyFieldLines.join('\n'));
+    }
+
+    // 3. Synonyms
+    const synonymLines = tableAgentInstructions.synonyms
+        .filter(s => !isExcluded(s.dependsOn, excludedColumns))
+        .map(s => `- "${s.terms.join('" / "')}" → ${s.sql}`);
+
+    if (synonymLines.length > 0) {
+        sections.push('SYNONYMS:\n' + synonymLines.join('\n'));
+    }
+
+    // 4. Filtering and Grouping
+    const guidanceLines = tableAgentInstructions.filteringGuidance
+        .filter(g => !isExcluded(g.dependsOn, excludedColumns))
+        .map(g => `- ${g.text}`);
+
+    if (guidanceLines.length > 0) {
+        sections.push('FILTERING AND GROUPING:\n' + guidanceLines.join('\n'));
+    }
+
+    // 5. Event Vocabulary
+    const vocabParts = [];
+    const autoEvents = tableAgentInstructions.eventVocabulary.autoCollectedAndEnhanced
+        .filter(e => !excludedEvents.includes(e));
+    if (autoEvents.length > 0) {
+        vocabParts.push(`Auto-collected and enhanced measurement: ${autoEvents.join(', ')}`);
+    }
+
+    if (!isExcluded(['ecommerce'], excludedColumns)) {
+        const ecomEvents = tableAgentInstructions.eventVocabulary.ecommerce
+            .filter(e => !excludedEvents.includes(e));
+        if (ecomEvents.length > 0) {
+            vocabParts.push(`Ecommerce (recommended): ${ecomEvents.join(', ')}`);
+        }
+    }
+
+    if (vocabParts.length > 0) {
+        sections.push('COMMON EVENT NAMES:\n' + vocabParts.join('\n'));
+    }
+
+    // 6. Table Features
+    const featureLines = [
+        'Combines daily, intraday, and fresh exports; the best available version of each event is used.',
+        'Incremental updates: non-final data is replaced with the latest available data on every run.',
+        'Promotes key fields (e.g. page_location, session_id) to top-level columns for faster queries.',
+        'Session-level fields: landing_page, user_id resolution, and configurable session parameters.',
+    ];
+    sections.push('TABLE FEATURES:\n' + featureLines.map(f => `- ${f}`).join('\n'));
+
+    // 7. Package Attribution
+    sections.push(`${constants.TABLE_DESCRIPTION_SUFFIX}\n${constants.TABLE_DESCRIPTION_DOCUMENTATION_LINK}`);
+
+    // 8. Config JSON dump
+    const configJson = JSON.stringify(
+        Object.fromEntries(
+            Object.entries(config).filter(([key]) => !key.startsWith('default') && key !== 'dataformTableConfig')
+        ),
+        null,
+        2
+    );
+    sections.push(`The last full table refresh was done using this configuration:\n${configJson}`);
+
+    return sections.join('\n\n');
+};
+
 module.exports = {
     columnDescriptions,
     getColumnDescriptions,
+    getTableDescription,
     composeDescription,
     getLineageText,
     buildConfigNotes,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.3.2-dev.0",
+  "version": "0.3.2-dev.2",
   "description": "",
   "main": "index.js",
   "files": [

package/tables/ga4EventsEnhanced.js

@@ -321,28 +321,7 @@ ${excludedEventsSQL}`,
 const createEnhancedEventsTable = (dataformPublish, config) => {
     const mergedConfig = utils.mergeSQLConfigurations(defaultConfig, config);
 
-    const tableDescription = `GA4 Events Enhanced 
-
-- Combines daily (processed) and intraday exports so the best available version of each event is always used.
-- Incremental updates: All data with "data_is_final" flag set to false is deleted and replaced with the latest available data on every run (supports any schedule: daily, hourly, or custom).
-- Keeps the flexible schema of the original export while promoting key fields (e.g. page_location, session_id) to columns for faster queries.
-- Partitioned by event_date and clustered for optimal query performance.
-- Event parameter handling: promote params to columns or exclude by name.
-- Session-level fields: landing_page, fixed user_id, and configurable session parameters.
-- Other improvements and refinements based on configuration
-
-${constants.TABLE_DESCRIPTION_SUFFIX}
-${constants.TABLE_DESCRIPTION_DOCUMENTATION_LINK}
-
-The last full table refresh was done using this configuration:
-${JSON.stringify(
-  Object.fromEntries(
-    // don't display the default arrays here, their contents are included in the main arrays via the mergeSQLConfigurations function
-    Object.entries(mergedConfig).filter(([key]) => !key.startsWith('default'))
-  ),
-  null,
-  2
-)}`;
+    const tableDescription = documentation.getTableDescription(mergedConfig);
 
     // the defaults for the dataform table config
     const defaultDataformTableConfig = {