@tech4good/angular-schematics 21.4.0 → 21.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/package.json +1 -1
  3. package/src/ng-add/files/.firebaserc.template +2 -1
  4. package/src/ng-add/files/bigquery/queries/active_time_by_route.sql.template +34 -0
  5. package/src/ng-add/files/bigquery/scripts/export_entities.py.template +143 -0
  6. package/src/ng-add/files/bigquery/scripts/run_query.py.template +167 -0
  7. package/src/ng-add/files/bigquery/views/events_flat.sql.template +60 -0
  8. package/src/ng-add/files/bigquery/views/logs_latest.sql.template +34 -0
  9. package/src/ng-add/files/docs-infra/analytics-guide.md.template +170 -0
  10. package/src/ng-add/files/firebase.logs.json.template +5 -0
  11. package/src/ng-add/files/firestore.indexes.json.template +89 -0
  12. package/src/ng-add/files/firestore.logs.rules.spec.ts.template +100 -0
  13. package/src/ng-add/files/firestore.logs.rules.template +8 -0
  14. package/src/ng-add/files/jest.config.ts.template +1 -1
  15. package/src/ng-add/files/package.json.template +4 -3
  16. package/src/ng-add/files/src/app/core/analytics/log.service.ts.template +26 -4
  17. package/src/ng-add/files/src/app/core/analytics/time-analytics.service.spec.ts.template +145 -2
  18. package/src/ng-add/files/src/app/core/analytics/time-analytics.service.ts.template +355 -149
  19. package/src/ng-add/files/src/app/core/auth/guards/require-auth-and-consent.guard.ts.template +3 -1
  20. package/src/ng-add/files/src/app/first-time/consent/consent.component.html.template +172 -48
  21. package/src/ng-add/files/src/app/first-time/consent/consent.component.scss.template +31 -0
  22. package/src/ng-add/files/src/app/first-time/consent/consent.component.spec.ts.template +31 -3
  23. package/src/ng-add/files/src/app/first-time/consent/consent.component.ts.template +12 -11
  24. package/src/ng-add/files/src/app/first-time/consent/consent.guard.ts.template +0 -6
  25. package/src/ng-add/files/src/app/first-time/early-access/early-access.guard.ts.template +7 -1
  26. package/src/ng-add/files/src/app/first-time/first-time-routing.spec.ts.template +7 -6
package/CHANGELOG.md CHANGED
@@ -1,3 +1,20 @@
1
+ ## [21.5.0](https://github.com/tech4good-lab/angular-schematics/compare/v21.4.0...v21.5.0) (2026-07-22)
2
+
3
+ ### Features
4
+
5
+ * **ng-add:** add Firestore logging environment and security rules templates ([d5151c0](https://github.com/tech4good-lab/angular-schematics/commit/d5151c0f3ac63d5cd935cdf07a9fd970ac27f16a))
6
+ * **ng-add:** scaffold BigQuery views, queries, guide, and CLI scripts ([b4c2b7b](https://github.com/tech4good-lab/angular-schematics/commit/b4c2b7b44d6144cb9bf07be43431ef288d460206))
7
+ * **ng-add:** support object serialization in telemetry interactions ([ac14a3a](https://github.com/tech4good-lab/angular-schematics/commit/ac14a3ac290b12528eb52bc1ecb4233b93f3bc9a))
8
+ * **ng-add:** support revisitable consent page ([602ee5e](https://github.com/tech4good-lab/angular-schematics/commit/602ee5eaa25249e324b70aea2dce75d1c3a2b67b))
9
+ * **ng-add:** update consent component template and legal text ([bdf6926](https://github.com/tech4good-lab/angular-schematics/commit/bdf69262974346530f5137535042f38b4b1c54e9))
10
+
11
+ ### Code Refactoring
12
+
13
+ * **ng-add:** update core time analytics service and analytics guide templates ([7781f43](https://github.com/tech4good-lab/angular-schematics/commit/7781f43ba79f20e98b3fe73bea2854f1bdd8e89c))
14
+
15
+ ### Build System
16
+
17
+ * **ng-add:** bump template dependency for release ([5b644cd](https://github.com/tech4good-lab/angular-schematics/commit/5b644cd9990e0eb8b9c636a35ab1be8c49877464))
1
18
  ## [21.4.0](https://github.com/tech4good-lab/angular-schematics/compare/v21.3.0...v21.4.0) (2026-07-08)
2
19
 
3
20
  ### Features
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tech4good/angular-schematics",
3
- "version": "21.4.0",
3
+ "version": "21.5.0",
4
4
  "description": "A set of schematics for generating modules, containers, components, and entities",
5
5
  "scripts": {
6
6
  "build": "tsc -p tsconfig.json",
@@ -2,7 +2,8 @@
2
2
  "projects": {
3
3
  "default": "<%= dasherize(project) %>-dev",
4
4
  "dev": "<%= dasherize(project) %>-dev",
5
- "prod": "<%= dasherize(project) %>-prod"
5
+ "prod": "<%= dasherize(project) %>-prod",
6
+ "log": "<%= dasherize(project) %>-log"
6
7
  },
7
8
  "targets": {},
8
9
  "etags": {}
@@ -0,0 +1,34 @@
1
+ -- Query: active_time_by_route
2
+ -- Description: Analyzes the distribution of active engagement time (in seconds) spent
3
+ -- on different routes. Computes total visits, mean active seconds, and percentiles
4
+ -- (Min, Q1, Median, Q3, Max) for each route.
5
+
6
+ WITH route_visit_times AS (
7
+ SELECT
8
+ route,
9
+ -- Convert milliseconds to seconds
10
+ total_active_ms / 1000.0 AS active_seconds
11
+ FROM
12
+ `analytics.events_flat`
13
+ WHERE
14
+ event_type = 'route_visit'
15
+ AND total_active_ms > 0
16
+ )
17
+
18
+ SELECT
19
+ route,
20
+ COUNT(*) AS total_visits,
21
+ ROUND(AVG(active_seconds), 2) AS mean_active_seconds,
22
+
23
+ -- APPROX_QUANTILES returns a 5-element array: [Min, 25th (Q1), 50th (Median), 75th (Q3), Max]
24
+ ROUND(APPROX_QUANTILES(active_seconds, 4)[OFFSET(0)], 2) AS min_active_seconds,
25
+ ROUND(APPROX_QUANTILES(active_seconds, 4)[OFFSET(1)], 2) AS q1_active_seconds,
26
+ ROUND(APPROX_QUANTILES(active_seconds, 4)[OFFSET(2)], 2) AS median_active_seconds,
27
+ ROUND(APPROX_QUANTILES(active_seconds, 4)[OFFSET(3)], 2) AS q3_active_seconds,
28
+ ROUND(APPROX_QUANTILES(active_seconds, 4)[OFFSET(4)], 2) AS max_active_seconds
29
+ FROM
30
+ route_visit_times
31
+ GROUP BY
32
+ route
33
+ ORDER BY
34
+ total_visits DESC;
@@ -0,0 +1,143 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ Firestore Entity Exporter
4
+
5
+ This script exports any specified Firestore collection to a local JSON or CSV file.
6
+ Useful for pulling low-volume, PII-sensitive collections (like user responses)
7
+ on-demand to run Jupyter notebook analyses without storing them persistently in BigQuery.
8
+
9
+ Usage:
10
+ # Export to JSON
11
+ python bigquery/scripts/export_entities.py --collection <collection-name> --output data.json
12
+
13
+ # Export to CSV
14
+ python bigquery/scripts/export_entities.py --collection <collection-name> --output data.csv
15
+ """
16
+
17
+ import argparse
18
+ import csv
19
+ import json
20
+ import os
21
+ import sys
22
+ from datetime import datetime
23
+
24
+ try:
25
+ import firebase_admin
26
+ from firebase_admin import credentials, firestore
27
+ from google.api_core.datetime_helpers import DatetimeWithNanoseconds
28
+ except ImportError:
29
+ print("Error: Required packages missing. Please install dependencies:")
30
+ print(" pip install firebase-admin google-api-core")
31
+ sys.exit(1)
32
+
33
+
34
+ def parse_value(val):
35
+ """Recursively converts Firestore Timestamp objects and other non-JSON-serializable types."""
36
+ if isinstance(val, DatetimeWithNanoseconds):
37
+ return val.isoformat()
38
+ elif isinstance(val, datetime):
39
+ return val.isoformat()
40
+ elif isinstance(val, dict):
41
+ return {k: parse_value(v) for k, v in val.items()}
42
+ elif isinstance(val, list):
43
+ return [parse_value(x) for x in val]
44
+ return val
45
+
46
+
47
+ def main():
48
+ parser = argparse.ArgumentParser(description="Export Firestore collections to JSON or CSV.")
49
+ parser.add_argument(
50
+ "--collection",
51
+ required=True,
52
+ help="Name of the Firestore collection to export",
53
+ )
54
+ parser.add_argument(
55
+ "--output",
56
+ required=True,
57
+ help="Local file path for the export (must end in .json or .csv)",
58
+ )
59
+ parser.add_argument(
60
+ "--credentials",
61
+ help="Path to the Firebase service account key JSON file. "
62
+ "If omitted, falls back to GOOGLE_APPLICATION_CREDENTIALS env var.",
63
+ )
64
+ args = parser.parse_args()
65
+
66
+ # 1. Initialize Firebase App
67
+ if not firebase_admin._apps:
68
+ try:
69
+ if args.credentials:
70
+ cred = credentials.Certificate(args.credentials)
71
+ firebase_admin.initialize_app(cred)
72
+ elif os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):
73
+ cred = credentials.Certificate(os.environ["GOOGLE_APPLICATION_CREDENTIALS"])
74
+ firebase_admin.initialize_app(cred)
75
+ else:
76
+ # Attempt to load Application Default Credentials
77
+ firebase_admin.initialize_app()
78
+ except Exception as e:
79
+ print("Authentication Error: Could not initialize Firebase Admin SDK.")
80
+ print("Please either:")
81
+ print(" 1. Download a service account JSON file from Firebase Console -> Project Settings -> Service accounts.")
82
+ print(" 2. Pass it using --credentials /path/to/key.json")
83
+ print(" 3. Or run: export GOOGLE_APPLICATION_CREDENTIALS='/path/to/key.json'")
84
+ print(f"Details: {e}")
85
+ sys.exit(1)
86
+
87
+ db = firestore.client()
88
+ collection_ref = db.collection(args.collection)
89
+
90
+ print(f"Fetching documents from collection '{args.collection}'...")
91
+ docs = list(collection_ref.stream())
92
+ print(f"Retrieved {len(docs)} documents.")
93
+
94
+ if not docs:
95
+ print("No documents found. Exiting.")
96
+ return
97
+
98
+ # 2. Extract and format data
99
+ flat_data = []
100
+ headers = set()
101
+
102
+ for doc in docs:
103
+ doc_dict = doc.to_dict()
104
+ doc_dict["__id"] = doc.id # Include primary document ID
105
+ parsed_dict = parse_value(doc_dict)
106
+ flat_data.append(parsed_dict)
107
+ headers.update(parsed_dict.keys())
108
+
109
+ # Sort headers to ensure consistent column ordering in CSV, placing __id first
110
+ headers = sorted(list(headers))
111
+ if "__id" in headers:
112
+ headers.remove("__id")
113
+ headers.insert(0, "__id")
114
+
115
+ # 3. Write output
116
+ output_path = args.output
117
+ ext = os.path.splitext(output_path)[1].lower()
118
+
119
+ if ext == ".json":
120
+ with open(output_path, "w", encoding="utf-8") as f:
121
+ json.dump(flat_data, f, indent=2, ensure_ascii=False)
122
+ print(f"Successfully exported data to JSON: {output_path}")
123
+
124
+ elif ext == ".csv":
125
+ with open(output_path, "w", encoding="utf-8", newline="") as f:
126
+ writer = csv.DictWriter(f, fieldnames=headers, extrasaction="ignore")
127
+ writer.writeheader()
128
+ for row in flat_data:
129
+ csv_row = {}
130
+ for k, v in row.items():
131
+ if isinstance(v, (dict, list)):
132
+ csv_row[k] = json.dumps(v, ensure_ascii=False)
133
+ else:
134
+ csv_row[k] = v
135
+ writer.writerow(csv_row)
136
+ print(f"Successfully exported data to CSV: {output_path}")
137
+ else:
138
+ print(f"Error: Unknown output format '{ext}'. Must be .json or .csv.")
139
+ sys.exit(1)
140
+
141
+
142
+ if __name__ == "__main__":
143
+ main()
@@ -0,0 +1,167 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ BigQuery Query Runner
4
+
5
+ Runs a specified SQL query file against Google BigQuery and exports the results
6
+ to JSON, CSV, or a local SQLite database.
7
+
8
+ Usage:
9
+ # Export to JSON
10
+ python bigquery/scripts/run_query.py --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output output.json
11
+
12
+ # Export to CSV
13
+ python bigquery/scripts/run_query.py --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output output.csv
14
+
15
+ # Export to SQLite
16
+ python bigquery/scripts/run_query.py --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output metrics.db --table active_time
17
+ """
18
+
19
+ import argparse
20
+ import csv
21
+ import json
22
+ import os
23
+ import sqlite3
24
+ import sys
25
+
26
+ try:
27
+ from google.cloud import bigquery
28
+ except ImportError:
29
+ print("Error: Required package 'google-cloud-bigquery' missing. Please install it:")
30
+ print(" pip install google-cloud-bigquery")
31
+ sys.exit(1)
32
+
33
+
34
+ def main():
35
+ parser = argparse.ArgumentParser(description="Run BigQuery SQL queries and export results.")
36
+ parser.add_argument(
37
+ "--query",
38
+ required=True,
39
+ help="Path to the SQL file to run (e.g., bigquery/queries/active_time_by_route.sql)",
40
+ )
41
+ parser.add_argument(
42
+ "--output",
43
+ required=True,
44
+ help="Local file path for the export (must end in .json, .csv, or .db/.sqlite)",
45
+ )
46
+ parser.add_argument(
47
+ "--project",
48
+ required=True,
49
+ help="GCP Project ID to run the query against (e.g., <your-gcp-project-id>)",
50
+ )
51
+ parser.add_argument(
52
+ "--table",
53
+ help="Target table name when exporting to a SQLite database. Required if output is SQLite.",
54
+ )
55
+ parser.add_argument(
56
+ "--credentials",
57
+ help="Path to the Google service account key JSON file. "
58
+ "If omitted, falls back to GOOGLE_APPLICATION_CREDENTIALS env var.",
59
+ )
60
+ args = parser.parse_args()
61
+
62
+ # 1. Validate inputs
63
+ if not os.path.exists(args.query):
64
+ print(f"Error: Query file not found at {args.query}")
65
+ sys.exit(1)
66
+
67
+ output_ext = os.path.splitext(args.output)[1].lower()
68
+ is_sqlite = output_ext in [".db", ".sqlite", ".sqlite3"]
69
+ if is_sqlite and not args.table:
70
+ print("Error: --table argument is required when exporting to a SQLite database.")
71
+ sys.exit(1)
72
+
73
+ with open(args.query, "r", encoding="utf-8") as f:
74
+ query_string = f.read()
75
+
76
+ # 2. Initialize BigQuery Client
77
+ try:
78
+ if args.credentials:
79
+ client = bigquery.Client.from_service_account_json(args.credentials, project=args.project)
80
+ else:
81
+ client = bigquery.Client(project=args.project)
82
+ except Exception as e:
83
+ print("Authentication Error: Could not initialize BigQuery Client.")
84
+ print("Please check your credentials or run:")
85
+ print(" export GOOGLE_APPLICATION_CREDENTIALS='/path/to/key.json'")
86
+ print(f"Details: {e}")
87
+ sys.exit(1)
88
+
89
+ # 3. Execute Query
90
+ print(f"Executing query '{args.query}' in project '{args.project}'...")
91
+ try:
92
+ query_job = client.query(query_string)
93
+ results = query_job.result() # Waits for query to complete
94
+ print("Query completed successfully. Fetching rows...")
95
+ except Exception as e:
96
+ print(f"Query Execution Error: {e}")
97
+ sys.exit(1)
98
+
99
+ # Convert rows to a list of dicts for export
100
+ rows = [dict(row) for row in results]
101
+ if not rows:
102
+ print("Query returned 0 rows. Exiting.")
103
+ return
104
+
105
+ # 4. Write Output
106
+ if output_ext == ".json":
107
+ with open(args.output, "w", encoding="utf-8") as f:
108
+ json.dump(rows, f, indent=2, default=str)
109
+ print(f"Successfully exported {len(rows)} rows to JSON: {args.output}")
110
+
111
+ elif output_ext == ".csv":
112
+ headers = list(rows[0].keys())
113
+ with open(args.output, "w", encoding="utf-8", newline="") as f:
114
+ writer = csv.DictWriter(f, fieldnames=headers)
115
+ writer.writeheader()
116
+ for row in rows:
117
+ # Stringify nested structures
118
+ row_clean = {k: (json.dumps(v) if isinstance(v, (dict, list)) else v) for k, v in row.items()}
119
+ writer.writerow(row_clean)
120
+ print(f"Successfully exported {len(rows)} rows to CSV: {args.output}")
121
+
122
+ elif is_sqlite:
123
+ headers = list(rows[0].keys())
124
+ conn = sqlite3.connect(args.output)
125
+ cursor = conn.cursor()
126
+
127
+ # Dynamic table creation based on BigQuery results schema
128
+ # Maps common types, defaults everything else to TEXT
129
+ columns_def = []
130
+ for key, val in rows[0].items():
131
+ if isinstance(val, int):
132
+ col_type = "INTEGER"
133
+ elif isinstance(val, float):
134
+ col_type = "REAL"
135
+ else:
136
+ col_type = "TEXT"
137
+ columns_def.append(f"`{key}` {col_type}")
138
+
139
+ create_table_sql = f"CREATE TABLE IF NOT EXISTS `{args.table}` ({', '.join(columns_def)})"
140
+ cursor.execute(create_table_sql)
141
+
142
+ # Insert rows
143
+ insert_placeholders = ", ".join(["?"] * len(headers))
144
+ insert_sql = f"INSERT INTO `{args.table}` ({', '.join([f'`{h}`' for h in headers])}) VALUES ({insert_placeholders})"
145
+
146
+ insert_data = []
147
+ for row in rows:
148
+ row_data = []
149
+ for header in headers:
150
+ val = row[header]
151
+ if isinstance(val, (dict, list)):
152
+ row_data.append(json.dumps(val))
153
+ else:
154
+ row_data.append(val)
155
+ insert_data.append(tuple(row_data))
156
+
157
+ cursor.executemany(insert_sql, insert_data)
158
+ conn.commit()
159
+ conn.close()
160
+ print(f"Successfully imported {len(rows)} rows into SQLite table '{args.table}': {args.output}")
161
+ else:
162
+ print(f"Error: Unknown output format '{output_ext}'. Must be .json, .csv, or .db/.sqlite.")
163
+ sys.exit(1)
164
+
165
+
166
+ if __name__ == "__main__":
167
+ main()
@@ -0,0 +1,60 @@
1
+ -- View: events_flat
2
+ -- Description: Flattens the nested time segments and interactions arrays in the latest logs
3
+ -- into a single clean event-stream format with one row per event.
4
+ -- Preserves the raw chronological time segments array for advanced engagement pattern analysis.
5
+
6
+ CREATE OR REPLACE VIEW `analytics.events_flat` AS
7
+ -- 1. Route visits (emitted when a route is closed/navigated away from)
8
+ SELECT
9
+ document_id,
10
+ __userId AS user_id,
11
+ __sessionId AS session_id,
12
+ __deviceId AS device_id,
13
+ route,
14
+ _createdAt,
15
+ _updatedAt,
16
+ 'route_visit' AS event_type,
17
+ startTime AS timestamp,
18
+ CAST(NULL AS STRING) AS category,
19
+ CAST(NULL AS STRING) AS action,
20
+ CAST(NULL AS STRING) AS properties,
21
+ -- Preserves the chronological pattern of active/rest intervals (JSON array)
22
+ time_segments,
23
+ COALESCE(
24
+ (SELECT SUM(CAST(JSON_VALUE(t, '$.total') AS INT64))
25
+ FROM UNNEST(JSON_QUERY_ARRAY(time_segments)) t
26
+ WHERE JSON_VALUE(t, '$.type') = 'active'),
27
+ 0
28
+ ) AS total_active_ms,
29
+ COALESCE(
30
+ (SELECT SUM(CAST(JSON_VALUE(t, '$.total') AS INT64))
31
+ FROM UNNEST(JSON_QUERY_ARRAY(time_segments)) t
32
+ WHERE JSON_VALUE(t, '$.type') = 'rest'),
33
+ 0
34
+ ) AS total_rest_ms
35
+ FROM
36
+ `analytics.logs_latest`
37
+
38
+ UNION ALL
39
+
40
+ -- 2. Interactions (discrete click, focus, and submission events)
41
+ SELECT
42
+ document_id,
43
+ __userId AS user_id,
44
+ __sessionId AS session_id,
45
+ __deviceId AS device_id,
46
+ route,
47
+ _createdAt,
48
+ _updatedAt,
49
+ 'interaction' AS event_type,
50
+ CAST(JSON_VALUE(ix, '$[0]') AS INT64) AS timestamp,
51
+ JSON_VALUE(ix, '$[1]') AS category,
52
+ JSON_VALUE(ix, '$[2]') AS action,
53
+ JSON_VALUE(ix, '$[3]') AS properties,
54
+ -- Null for interactions
55
+ CAST(NULL AS STRING) AS time_segments,
56
+ CAST(NULL AS INT64) AS total_active_ms,
57
+ CAST(NULL AS INT64) AS total_rest_ms
58
+ FROM
59
+ `analytics.logs_latest`,
60
+ UNNEST(JSON_QUERY_ARRAY(interactions)) ix;
@@ -0,0 +1,34 @@
1
+ -- View: logs_latest
2
+ -- Description: Extracts the latest version of each log document from the raw changelog table,
3
+ -- deduplicating documents that have been updated (e.g., via sendBeacon writes).
4
+
5
+ CREATE OR REPLACE VIEW `analytics.logs_latest` AS
6
+ SELECT
7
+ document_id,
8
+ timestamp AS bq_sync_timestamp,
9
+ operation,
10
+ JSON_VALUE(data, '$.__userId') AS __userId,
11
+ JSON_VALUE(data, '$.__sessionId') AS __sessionId,
12
+ JSON_VALUE(data, '$.__deviceId') AS __deviceId,
13
+ JSON_VALUE(data, '$.route') AS route,
14
+ CAST(JSON_VALUE(data, '$.startTime') AS INT64) AS startTime,
15
+ JSON_QUERY(data, '$.time') AS time_segments,
16
+ JSON_QUERY(data, '$.interactions') AS interactions,
17
+ CAST(JSON_VALUE(data, '$._createdAt') AS INT64) AS _createdAt,
18
+ CAST(JSON_VALUE(data, '$._updatedAt') AS INT64) AS _updatedAt
19
+ FROM (
20
+ SELECT
21
+ document_id,
22
+ timestamp,
23
+ operation,
24
+ data,
25
+ ROW_NUMBER() OVER (
26
+ PARTITION BY document_id
27
+ ORDER BY timestamp DESC
28
+ ) AS rn
29
+ FROM
30
+ `analytics.logs_raw_changelog`
31
+ )
32
+ WHERE
33
+ rn = 1
34
+ AND operation != 'DELETE';
@@ -0,0 +1,170 @@
1
+ # Analytics & Data Warehousing Guide
2
+
3
+ This document describes the telemetry logging and data warehousing architecture. It explains how user interactions are tracked, how data is synchronized to BigQuery, and how to query this data for research and analysis.
4
+
5
+ ---
6
+
7
+ ## 1. Architecture Overview
8
+
9
+ To support both real-time client-side tracking and complex research analysis (like NLP, statistical distribution, and LLM classifications), the analytics infrastructure uses a two-tier database strategy:
10
+
11
+ ```
12
+ ┌───────────────────────────────────────────────┐
13
+ │ Browser (Angular Client) │
14
+ │ Emits route-changes and interaction events │
15
+ └──────────────────────┬────────────────────────┘
16
+ │
17
+ ┌─────────────▼─────────────┐
18
+ │ Firestore: <log-project> │
19
+ │ logs/{logId} │
20
+ └─────────────┬─────────────┘
21
+ │ (Firebase BigQuery Export)
22
+ ┌─────────────▼─────────────┐
23
+ │ Google BigQuery │
24
+ │ analytics.logs_latest │ ← SQL View (latest state)
25
+ │ analytics.events_flat │ ← SQL View (event rows)
26
+ └───────────────────────────┘
27
+ ```
28
+
29
+ 1. **Firestore (`<log-project>` project)**: Serves as the high-volume ingestion layer. The client writes telemetry events to the `logs` collection. Firestore handles unreliable client networks, localStorage caching, retries, and browser beacon events.
30
+ 2. **BigQuery**: Serves as the analysis database. Telemetry documents are streamed in real-time to BigQuery using the **Firestore BigQuery Export** extension. Researchers run SQL queries and Python scripts against BigQuery to calculate metrics.
31
+ 3. **On-Demand Entity Export**: Direct application entities (like user answers or transaction records) are exported to BigQuery or Python on-demand using a Python script. This preserves privacy and makes user deletion requests trivial.
32
+
33
+ ---
34
+
35
+ ## 2. In-App Telemetry & Logging
36
+
37
+ Client-side telemetry is managed by the `TimeAnalyticsService`.
38
+
39
+ ### 2.1 Log Document Structure
40
+ Telemetry logs are written to Firestore under the collection `logs` with document IDs matching the format `${__sessionId}__${startTime}`. Each document represents a single page/route visit or registered UI state segment and contains:
41
+
42
+ ```json
43
+ {
44
+ "__userId": "user_123",
45
+ "__sessionId": "session_abc",
46
+ "__deviceId": "device_xyz",
47
+ "route": "/practice",
48
+ "state": { "problemId": "prob_456" },
49
+ "startTime": 1720000000000,
50
+ "time": [
51
+ { "type": "active", "total": 5200 },
52
+ { "type": "rest", "total": 12000 }
53
+ ],
54
+ "interactions": [
55
+ [1720000001000, "problem", "hint_click", "{\"hintIndex\":1}"],
56
+ [1720000005000, "problem", "submit_attempt", "{\"attemptText\":\"42\"}"]
57
+ ],
58
+ "_createdAt": 1720000000000,
59
+ "_updatedAt": 1720000006000
60
+ }
61
+ ```
62
+
63
+ * **`route` & `state`**: Tracks current URL route and optional registered UI state dimensions (e.g. `state.problemId`).
64
+ * **`time` array**: Tracks accumulated active and rest time on the route/state segment (measured using window focus, visibility, and mouse/keyboard activity).
65
+ * **`interactions` array**: Custom events tracked during the visit. Each interaction is a tuple: `[timestamp, category, action, JSON_stringified_properties]`.
66
+ * **`_createdAt` & `_updatedAt`**: Client-side epoch timestamps (in milliseconds) capturing when the log was opened and when it was last written (including beacon overwrites on tab close).
67
+
68
+ ### 2.2 Registering UI State Dimensions (`registerState`)
69
+ By default, logs are flushed when route or user changes. For components where users remain on a single route across multiple distinct items (such as solving math problems on `/practice`), components register state signals with `TimeAnalyticsService`:
70
+
71
+ ```typescript
72
+ // In component constructor
73
+ this.analytics.registerState(
74
+ 'problemId',
75
+ computed(() => this.currentProgress()?.__currentProblemId ?? null),
76
+ this.injector
77
+ );
78
+ ```
79
+
80
+ * **Behavior:** When the registered state signal value changes, `TimeAnalyticsService` flushes the current log segment to Firestore and starts a new segment for the new state value.
81
+ * **Cleanup:** State registrations are automatically cleaned up when the component is destroyed via `DestroyRef`.
82
+ * **Usage Rule:** Register state sparingly — only for dimensions requiring first-class time segmentation (e.g., `problemId`). General user interactions should be logged via `trackInteraction()`.
83
+
84
+ ---
85
+
86
+ ## 3. Configuring BigQuery Sync
87
+
88
+ To stream logs from Firestore to BigQuery, you must install the **Firestore BigQuery Export** extension on your logs Firebase project.
89
+
90
+ ### 3.1 Installation Steps
91
+ 1. In the Firebase Console, navigate to **Extensions**.
92
+ 2. Search for **Firestore BigQuery Export** and click **Install**.
93
+ 3. Configure the extension with the following parameters:
94
+ - **Collection path**: `logs`
95
+ - **Dataset ID**: `analytics`
96
+ - **Table ID**: `logs_raw`
97
+ - **Cloud Function location**: Select the region closest to your Firestore database (e.g., `us-central1`).
98
+ 4. Give the generated service account permissions to write to BigQuery.
99
+
100
+ The extension will create a BigQuery dataset named `analytics` containing:
101
+ * `logs_raw_changelog`: A table that appends a new row on every Firestore document create, update, or delete.
102
+ * `logs_raw_schema`: Internal schema helper table.
103
+
104
+ ---
105
+
106
+ ## 4. Understanding the BigQuery Changelog
107
+
108
+ Because BigQuery is an append-only warehouse, the extension does not edit existing rows. If a user's browser updates a log document (e.g. adding time via `sendBeacon` when they close the tab), a new row is appended to `logs_raw_changelog`.
109
+
110
+ Each row in `logs_raw_changelog` has the following system columns:
111
+ * `document_id`: The Firestore document path (`logs/sessionId__startTime`).
112
+ * `timestamp`: When the write was synced to BigQuery.
113
+ * `operation`: The Firestore write operation: `IMPORT`, `CREATE`, `UPDATE`, or `DELETE`.
114
+ * `data`: A JSON string containing the full serialized Firestore document.
115
+
116
+ To query the current state of logs without duplicates, we use a SQL view that filters for the latest update per document.
117
+
118
+ ---
119
+
120
+ ## 5. SQL Views & Queries
121
+
122
+ The SQL view files are located in `/bigquery/views/` and `/bigquery/queries/`.
123
+
124
+ ### 5.1 No Project Prefixes Needed
125
+ Our SQL scripts are designed **relativity** (omitting the GCP Project ID prefix, e.g. using `analytics.events_flat` instead of `your_project.analytics.events_flat`).
126
+ This means they run cleanly out of the box in the BigQuery Web Console (which executes them in your active project workspace) or via terminal scripts.
127
+
128
+ ### 5.2 Deduplicating Logs (`logs_latest`)
129
+ The view `logs_latest` extracts only the latest state of each document:
130
+ ```sql
131
+ SELECT * FROM `analytics.logs_latest` LIMIT 10;
132
+ ```
133
+
134
+ ### 5.3 Flattening Telemetry (`events_flat`)
135
+ The `events_flat` view flattens time segments and interactions nested arrays into standard tabular rows. Every row in `events_flat` represents a discrete event, typed as either:
136
+ - `route_visit`: Emitted when the user navigates away from a route. Includes columns `total_active_ms`, `total_rest_ms`, and `time_segments` (a raw JSON array containing the chronological sequence of active/idle intervals for pattern analysis).
137
+ - `interaction`: Emitted for custom telemetry actions. Includes columns `category`, `action`, and `properties` (parsed JSON).
138
+
139
+ ---
140
+
141
+ ## 6. Running Queries & Exports via Terminal
142
+
143
+ To avoid browser copy-paste workflows and support data analyses, we provide terminal scripts inside `bigquery/scripts/`.
144
+
145
+ ### 6.1 Running SQL Queries (`run_query.py`)
146
+ Use `run_query.py` to run any SQL file from the terminal and export the results to **JSON**, **CSV**, or a local **SQLite database**.
147
+
148
+ ```bash
149
+ # Export active time statistics by route to a local JSON file
150
+ python bigquery/scripts/run_query.py --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output output.json
151
+
152
+ # Export active time statistics by route to a local CSV file
153
+ python bigquery/scripts/run_query.py --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output output.csv
154
+
155
+ # Export active time statistics by route directly into a local SQLite table named 'active_time'
156
+ python bigquery/scripts/run_query.py --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output stats.db --table active_time
157
+ ```
158
+
159
+ ### 6.2 On-Demand Entity Export (`export_entities.py`)
160
+ Application entities contain sensitive data and are not automatically synchronized to BigQuery. Use `export_entities.py` to pull these collections directly from Firestore to local files on-demand for research.
161
+
162
+ ```bash
163
+ # Export documents directly from Firestore to a local JSON file
164
+ python bigquery/scripts/export_entities.py --credentials path/to/key.json --collection <collection-name> --output data.json
165
+
166
+ # Export documents to a local CSV file (auto-flattens nested fields)
167
+ python bigquery/scripts/export_entities.py --credentials path/to/key.json --collection <collection-name> --output data.csv
168
+ ```
169
+
170
+ *Note: If `--credentials` is omitted, the scripts automatically search the environment for `GOOGLE_APPLICATION_CREDENTIALS` or Application Default Credentials.*
@@ -0,0 +1,5 @@
1
+ {
2
+ "firestore": {
3
+ "rules": "firestore.logs.rules"
4
+ }
5
+ }