@tech4good/angular-schematics 21.6.0 → 21.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +266 -1
- package/package.json +34 -7
- package/src/component/files/__name@dasherize__/__name@dasherize__.component.spec.ts.template +4 -5
- package/src/component/files/__name@dasherize__/__name@dasherize__.component.ts.template +1 -1
- package/src/entity/files/__name@dasherize__/__name@dasherize__.mock.ts.template +14 -6
- package/src/entity/files/__name@dasherize__/__name@dasherize__.model.ts.template +12 -6
- package/src/entity/files/__name@dasherize__/__name@dasherize__.store.ts.template +18 -10
- package/src/entity/index.js +4 -49
- package/src/entity/index.js.map +1 -1
- package/src/guard/files/__name@dasherize__.guard.spec.ts.template +0 -1
- package/src/ng-add/files/.github/workflows/angular-github-ci.yml.template +3 -0
- package/src/ng-add/files/.gitignore.template +3 -0
- package/src/ng-add/files/analytics/datasets.js.template +206 -0
- package/src/ng-add/files/analytics/entity_sources.js.template +58 -0
- package/src/ng-add/files/analytics/entity_sources.test.js.template +57 -0
- package/src/ng-add/files/analytics/lib/api_query.js.template +87 -0
- package/src/ng-add/files/analytics/lib/api_query.test.js.template +84 -0
- package/src/ng-add/files/analytics/lib/bigquery_query_compatibility.test.js.template +53 -0
- package/src/ng-add/files/analytics/lib/dataset_schema.js.template +53 -0
- package/src/ng-add/files/analytics/lib/dataset_schema.test.js.template +34 -0
- package/src/ng-add/files/analytics/lib/entity_sync.js.template +397 -0
- package/src/ng-add/files/analytics/lib/entity_sync.test.js.template +212 -0
- package/src/ng-add/files/analytics/lib/identifier_salt.js.template +57 -0
- package/src/ng-add/files/analytics/lib/identifier_salt.test.js.template +36 -0
- package/src/ng-add/files/analytics/lib/snapshot_files.js.template +46 -0
- package/src/ng-add/files/analytics/lib/snapshot_files.test.js.template +67 -0
- package/src/ng-add/files/analytics/lib/sqlite.js.template +92 -0
- package/src/ng-add/files/analytics/lib/upload_entities.js.template +81 -0
- package/src/ng-add/files/analytics/lib/upload_entities.test.js.template +86 -0
- package/src/ng-add/files/analytics/proxy.conf.json.template +7 -0
- package/src/ng-add/files/analytics/scripts/refresh_snapshot.js.template +518 -0
- package/src/ng-add/files/analytics/scripts/start.js.template +88 -0
- package/src/ng-add/files/analytics/server/server.js.template +212 -0
- package/src/ng-add/files/angular.json.template +80 -1
- package/src/ng-add/files/bigquery/queries/active_time_by_route.sql.template +27 -8
- package/src/ng-add/files/bigquery/queries/active_user_windows.sql.template +67 -0
- package/src/ng-add/files/bigquery/queries/dashboard_route_visits.sql.template +47 -0
- package/src/ng-add/files/bigquery/queries/data_quality.sql.template +323 -0
- package/src/ng-add/files/bigquery/queries/onboarding_funnel.sql.template +512 -0
- package/src/ng-add/files/bigquery/queries/onboarding_stage_observations.sql.template +134 -0
- package/src/ng-add/files/bigquery/queries/weekly_retention_cohorts.sql.template +90 -0
- package/src/ng-add/files/bigquery/scripts/export_entities.js.template +120 -0
- package/src/ng-add/files/bigquery/scripts/init_views.js.template +57 -0
- package/src/ng-add/files/bigquery/scripts/run_query.js.template +141 -0
- package/src/ng-add/files/bigquery/views/events_flat.sql.template +27 -12
- package/src/ng-add/files/bigquery/views/logs_latest.sql.template +16 -3
- package/src/ng-add/files/docs-infra/analytics-guide.md.template +270 -25
- package/src/ng-add/files/docs-infra/analytics-workbench-guide.md.template +307 -0
- package/src/ng-add/files/docs-infra/authentication-architecture.md.template +54 -24
- package/src/ng-add/files/docs-infra/email-infrastructure.md.template +43 -0
- package/src/ng-add/files/docs-infra/firestore-index-checker.md.template +87 -0
- package/src/ng-add/files/docs-infra/infrastructure-guide.md.template +108 -29
- package/src/ng-add/files/docs-infra/load-system-architecture.md.template +15 -1
- package/src/ng-add/files/docs-infra/querying-strategy.md.template +14 -0
- package/src/ng-add/files/docs-infra/stream-system-architecture.md.template +21 -7
- package/src/ng-add/files/eslint.config.js.template +22 -3
- package/src/ng-add/files/functions/package.json.template +1 -0
- package/src/ng-add/files/functions/src/email.ts.template +23 -0
- package/src/ng-add/files/jest.config.ts.template +3 -2
- package/src/ng-add/files/package.json.template +39 -16
- package/src/ng-add/files/projects/analytics/src/app/analytics-api.service.ts.template +130 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.html.template +77 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.scss.template +240 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.ts.template +96 -0
- package/src/ng-add/files/projects/analytics/src/app/app.routes.ts.template +11 -0
- package/src/ng-add/files/projects/analytics/src/app/dashboard-pages.spec.ts.template +151 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.html.template +39 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.ts.template +95 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.html.template +47 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.ts.template +120 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.html.template +40 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.ts.template +98 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.html.template +35 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.ts.template +84 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.html.template +42 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.ts.template +110 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/analytics-utils.ts.template +43 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/chart-theme.ts.template +42 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-layout.scss.template +239 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.html.template +76 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.scss.template +128 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.spec.ts.template +31 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.ts.template +153 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.html.template +16 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.scss.template +63 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.ts.template +52 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.html.template +5 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.scss.template +56 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.spec.ts.template +30 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.ts.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.html.template +9 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.scss.template +43 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.ts.template +62 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.html.template +9 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.scss.template +33 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.ts.template +82 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.html.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.scss.template +45 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.ts.template +55 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.html.template +18 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.scss.template +45 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.ts.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/vega-chart/vega-chart.component.ts.template +121 -0
- package/src/ng-add/files/projects/analytics/src/index.html.template +19 -0
- package/src/ng-add/files/projects/analytics/src/main.ts.template +16 -0
- package/src/ng-add/files/projects/analytics/src/styles.scss.template +28 -0
- package/src/ng-add/files/projects/analytics/tsconfig.app.json.template +13 -0
- package/src/ng-add/files/projects/analytics/tsconfig.spec.json.template +15 -0
- package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/admin/admin-panel/admin-settings/admin-settings.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.spec.ts.template +11 -12
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.ts.template +7 -7
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.spec.ts.template +8 -10
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.ts.template +3 -2
- package/src/ng-add/files/src/app/app.component.html.template +1 -1
- package/src/ng-add/files/src/app/app.component.scss.template +1 -1
- package/src/ng-add/files/src/app/app.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/app.component.ts.template +1 -2
- package/src/ng-add/files/src/app/app.config.ts.template +21 -15
- package/src/ng-add/files/src/app/auth-routing.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.spec.ts.template +46 -29
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.ts.template +27 -18
- package/src/ng-add/files/src/app/core/auth/auth.providers.spec.ts.template +112 -0
- package/src/ng-add/files/src/app/core/auth/auth.providers.ts.template +48 -0
- package/src/ng-add/files/src/app/core/auth/auth.service.ts.template +9 -23
- package/src/ng-add/files/src/app/core/auth/guards/require-admin.guard.spec.ts.template +3 -14
- package/src/ng-add/files/src/app/core/firebase/data.providers.prod.ts.template +21 -0
- package/src/ng-add/files/src/app/core/firebase/data.providers.ts.template +13 -0
- package/src/ng-add/files/src/app/core/local-storage/session.service.spec.ts.template +43 -0
- package/src/ng-add/files/src/app/core/local-storage/session.service.ts.template +7 -0
- package/src/ng-add/files/src/app/core/store/app-data.spec.ts.template +76 -0
- package/src/ng-add/files/src/app/core/store/auth/auth.mock.ts.template +19 -67
- package/src/ng-add/files/src/app/core/store/auth/auth.store.spec.ts.template +102 -12
- package/src/ng-add/files/src/app/core/store/auth/auth.store.ts.template +32 -237
- package/src/ng-add/files/src/app/core/store/user/user.mock.ts.template +17 -12
- package/src/ng-add/files/src/app/core/store/user/user.model.ts.template +6 -7
- package/src/ng-add/files/src/app/core/store/user/user.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/store/user-context/user-context.mock.ts.template +20 -8
- package/src/ng-add/files/src/app/core/store/user-context/user-context.model.ts.template +5 -6
- package/src/ng-add/files/src/app/core/store/user-context/user-context.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/store/user-private/user-private.mock.ts.template +20 -7
- package/src/ng-add/files/src/app/core/store/user-private/user-private.model.ts.template +5 -6
- package/src/ng-add/files/src/app/core/store/user-private/user-private.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/testing/app-mocks.ts.template +42 -0
- package/src/ng-add/files/src/app/first-time/consent/consent.component.html.template +2 -1
- package/src/ng-add/files/src/app/first-time/consent/consent.component.scss.template +7 -3
- package/src/ng-add/files/src/app/first-time/consent/consent.component.spec.ts.template +31 -11
- package/src/ng-add/files/src/app/first-time/consent/consent.component.ts.template +16 -1
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.spec.ts.template +15 -15
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.ts.template +11 -3
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.html.template +2 -1
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.scss.template +7 -3
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.spec.ts.template +20 -6
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.ts.template +2 -0
- package/src/ng-add/files/src/app/first-time/early-access/waitlist-view/waitlist-view.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.spec.ts.template +17 -1
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.ts.template +15 -0
- package/src/ng-add/files/src/app/first-time/first-time-routing.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.ts.template +15 -0
- package/src/ng-add/files/src/app/first-time/landing/landing.component.spec.ts.template +2 -0
- package/src/ng-add/files/src/app/first-time/login/login.component.spec.ts.template +21 -3
- package/src/ng-add/files/src/app/first-time/login/login.component.ts.template +26 -0
- package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.spec.ts.template +4 -2
- package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.ts.template +1 -1
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.spec.ts.template +4 -4
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/main/home/home.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/main/home/home.component.ts.template +1 -1
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.spec.ts.template +4 -4
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.spec.ts.template +2 -2
- package/src/ng-add/files/tsconfig.spec.json.template +3 -1
- package/src/ng-add/index.js +7 -0
- package/src/ng-add/index.js.map +1 -1
- package/src/shared/entity-registry.d.ts +8 -0
- package/src/shared/entity-registry.js +28 -0
- package/src/shared/entity-registry.js.map +1 -0
- package/.agents/skills/publish-npm/SKILL.md +0 -37
- package/.agents/skills/publish-npm/eval.yaml +0 -32
- package/.agents/skills/publish-npm/fixtures/package-v21.json.template +0 -8
- package/.husky/commit-msg +0 -1
- package/.husky/pre-commit +0 -1
- package/.t4g/agent-memory.db +0 -0
- package/src/component/index_spec.d.ts +0 -1
- package/src/component/index_spec.js +0 -68
- package/src/component/index_spec.js.map +0 -1
- package/src/entity/index_spec.d.ts +0 -1
- package/src/entity/index_spec.js +0 -153
- package/src/entity/index_spec.js.map +0 -1
- package/src/guard/index_spec.d.ts +0 -1
- package/src/guard/index_spec.js +0 -64
- package/src/guard/index_spec.js.map +0 -1
- package/src/ng-add/files/bigquery/scripts/export_entities.py.template +0 -143
- package/src/ng-add/files/bigquery/scripts/run_query.py.template +0 -167
- package/src/ng-add/files/src/app/core/firebase/cached-listeners.service.ts.template +0 -500
- package/src/ng-add/files/src/app/core/firebase/database.service.ts.template +0 -73
- package/src/ng-add/files/src/app/core/firebase/email-link-context.service.spec.ts.template +0 -59
- package/src/ng-add/files/src/app/core/firebase/email-link-context.service.ts.template +0 -147
- package/src/ng-add/files/src/app/core/firebase/firebase.mock.service.ts.template +0 -453
- package/src/ng-add/files/src/app/core/firebase/firebase.service.spec.ts.template +0 -116
- package/src/ng-add/files/src/app/core/firebase/firebase.service.ts.template +0 -639
- package/src/ng-add/files/src/app/core/firebase/simulate-load.spec.ts.template +0 -104
- package/src/ng-add/files/src/app/core/firebase/simulate-stream.spec.ts.template +0 -127
- package/src/ng-add/files/src/app/core/firebase/simulation.mock.ts.template +0 -38
- package/src/ng-add/files/src/app/core/firebase/stream-multiplexer.ts.template +0 -445
- package/src/ng-add/files/src/app/core/logging/logging.service.spec.ts.template +0 -118
- package/src/ng-add/files/src/app/core/logging/logging.service.ts.template +0 -168
- package/src/ng-add/files/src/app/core/store/app.model.ts.template +0 -21
- package/src/ng-add/files/src/app/core/store/app.store.spec.ts.template +0 -920
- package/src/ng-add/files/src/app/core/store/app.store.ts.template +0 -1320
- package/src/ng-add/files/src/app/core/store/batch-write.firebase.service.ts.template +0 -58
- package/src/ng-add/files/src/app/core/store/batch-write.mock.service.ts.template +0 -105
- package/src/ng-add/files/src/app/core/store/batch-write.service.ts.template +0 -126
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.html.template +0 -9
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.scss.template +0 -46
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.spec.ts.template +0 -69
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.ts.template +0 -39
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.spec.ts.template +0 -122
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.ts.template +0 -88
- package/src/ng-add/index_spec.d.ts +0 -1
- package/src/ng-add/index_spec.js +0 -108
- package/src/ng-add/index_spec.js.map +0 -1
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# Analytics & Data Warehousing Guide
|
|
2
2
|
|
|
3
|
-
This document describes the telemetry
|
|
3
|
+
This document describes the telemetry, application-entity, BigQuery, and local
|
|
4
|
+
snapshot architecture. It explains how events are tracked, how an allowlisted
|
|
5
|
+
user dimension is synchronized for admin-aware metrics, and how the developer
|
|
6
|
+
workbench receives immutable SQLite datasets.
|
|
4
7
|
|
|
5
8
|
---
|
|
6
9
|
|
|
@@ -18,17 +21,40 @@ To support both real-time client-side tracking and complex research analysis (li
|
|
|
18
21
|
│ Firestore: <log-project> │
|
|
19
22
|
│ logs/{logId} │
|
|
20
23
|
└─────────────┬─────────────┘
|
|
21
|
-
│ (
|
|
24
|
+
│ (Firestore BigQuery Export extension)
|
|
22
25
|
┌─────────────▼─────────────┐
|
|
23
26
|
│ Google BigQuery │
|
|
24
27
|
│ analytics.logs_latest │ ← SQL View (latest state)
|
|
25
28
|
│ analytics.events_flat │ ← SQL View (event rows)
|
|
29
|
+
│ analytics.dim_user │ ← Replaced each refresh from entities
|
|
26
30
|
└───────────────────────────┘
|
|
31
|
+
│
|
|
32
|
+
│ Parameterized metric queries (include/exclude admin)
|
|
33
|
+
┌─────────────▼─────────────┐
|
|
34
|
+
│ Local immutable SQLite │
|
|
35
|
+
│ .local/analytics/ │
|
|
36
|
+
└───────────────────────────┘
|
|
37
|
+
|
|
38
|
+
Application Firestore (`<%= dasherize(project) %>/users`) is read separately by the
|
|
39
|
+
refresh process. Its allowlisted, pseudonymous user dimension is cached locally,
|
|
40
|
+
uploaded to `analytics.dim_user`, and also included in the SQLite snapshot.
|
|
27
41
|
```
|
|
28
42
|
|
|
29
43
|
1. **Firestore (`<log-project>` project)**: Serves as the high-volume ingestion layer. The client writes telemetry events to the `logs` collection. Firestore handles unreliable client networks, localStorage caching, retries, and browser beacon events.
|
|
30
|
-
2. **
|
|
31
|
-
|
|
44
|
+
2. **Application Firestore (`<%= dasherize(project) %>`)**: Remains the source of truth
|
|
45
|
+
for user entities. The refresh incrementally derives only `user_key`, role,
|
|
46
|
+
access state, and source timestamps; raw names/emails never enter the cache.
|
|
47
|
+
3. **BigQuery**: Serves as the telemetry analysis database. It also receives a
|
|
48
|
+
full-replacement `analytics.dim_user` table before metrics run. That small
|
|
49
|
+
dimension exists solely so SQL can identify admin-attributed telemetry.
|
|
50
|
+
4. **Local SQLite snapshots**: Store typed, de-identified facts, aggregate
|
|
51
|
+
metrics, data-quality checks, and the user dimension for the localhost API.
|
|
52
|
+
The dashboard never queries BigQuery or Firestore directly.
|
|
53
|
+
|
|
54
|
+
Analytics scripts assume application project `<%= dasherize(project) %>` and
|
|
55
|
+
logging project `<%= dasherize(project) %>-logs` by default. Set
|
|
56
|
+
`ANALYTICS_APP_PROJECT_ID` and `ANALYTICS_LOG_PROJECT_ID` environment
|
|
57
|
+
variables when your Firebase project IDs use different names.
|
|
32
58
|
|
|
33
59
|
---
|
|
34
60
|
|
|
@@ -44,42 +70,42 @@ Telemetry logs are written to Firestore under the collection `logs` with documen
|
|
|
44
70
|
"__userId": "user_123",
|
|
45
71
|
"__sessionId": "session_abc",
|
|
46
72
|
"__deviceId": "device_xyz",
|
|
47
|
-
"route": "/
|
|
48
|
-
"state": { "
|
|
73
|
+
"route": "/home",
|
|
74
|
+
"state": { "itemId": "item_456" },
|
|
49
75
|
"startTime": 1720000000000,
|
|
50
76
|
"time": [
|
|
51
77
|
{ "type": "active", "total": 5200 },
|
|
52
78
|
{ "type": "rest", "total": 12000 }
|
|
53
79
|
],
|
|
54
80
|
"interactions": [
|
|
55
|
-
[1720000001000, "
|
|
56
|
-
[1720000005000, "
|
|
81
|
+
[1720000001000, "item", "open", "{\"itemType\":\"example\"}"],
|
|
82
|
+
[1720000005000, "item", "complete", "{\"source\":\"toolbar\"}"]
|
|
57
83
|
],
|
|
58
84
|
"_createdAt": 1720000000000,
|
|
59
85
|
"_updatedAt": 1720000006000
|
|
60
86
|
}
|
|
61
87
|
```
|
|
62
88
|
|
|
63
|
-
* **`route` & `state`**: Tracks current URL route and optional registered UI state dimensions (e.g. `state.
|
|
89
|
+
* **`route` & `state`**: Tracks current URL route and optional registered UI state dimensions (e.g. `state.itemId`).
|
|
64
90
|
* **`time` array**: Tracks accumulated active and rest time on the route/state segment (measured using window focus, visibility, and mouse/keyboard activity).
|
|
65
91
|
* **`interactions` array**: Custom events tracked during the visit. Each interaction is a tuple: `[timestamp, category, action, JSON_stringified_properties]`.
|
|
66
92
|
* **`_createdAt` & `_updatedAt`**: Client-side epoch timestamps (in milliseconds) capturing when the log was opened and when it was last written (including beacon overwrites on tab close).
|
|
67
93
|
|
|
68
94
|
### 2.2 Registering UI State Dimensions (`registerState`)
|
|
69
|
-
By default, logs are flushed when route or user changes. For components where users remain on a single route across multiple distinct items (such as
|
|
95
|
+
By default, logs are flushed when route or user changes. For components where users remain on a single route across multiple distinct items (such as browsing items on `/home`), components register state signals with `TimeAnalyticsService`:
|
|
70
96
|
|
|
71
97
|
```typescript
|
|
72
98
|
// In component constructor
|
|
73
99
|
this.analytics.registerState(
|
|
74
|
-
'
|
|
75
|
-
computed(() => this.currentProgress()?.
|
|
100
|
+
'itemId',
|
|
101
|
+
computed(() => this.currentProgress()?.__currentItemId ?? null),
|
|
76
102
|
this.injector
|
|
77
103
|
);
|
|
78
104
|
```
|
|
79
105
|
|
|
80
106
|
* **Behavior:** When the registered state signal value changes, `TimeAnalyticsService` flushes the current log segment to Firestore and starts a new segment for the new state value.
|
|
81
107
|
* **Cleanup:** State registrations are automatically cleaned up when the component is destroyed via `DestroyRef`.
|
|
82
|
-
* **Usage Rule:** Register state sparingly — only for dimensions requiring first-class time segmentation (e.g., `
|
|
108
|
+
* **Usage Rule:** Register state sparingly — only for dimensions requiring first-class time segmentation (e.g., `itemId`). General user interactions should be logged via `trackInteraction()`.
|
|
83
109
|
|
|
84
110
|
---
|
|
85
111
|
|
|
@@ -122,7 +148,7 @@ To query the current state of logs without duplicates, we use a SQL view that fi
|
|
|
122
148
|
The SQL view files are located in `/bigquery/views/` and `/bigquery/queries/`.
|
|
123
149
|
|
|
124
150
|
### 5.1 No Project Prefixes Needed
|
|
125
|
-
Our SQL scripts are designed **
|
|
151
|
+
Our SQL scripts are designed **relatively** (omitting the GCP Project ID prefix, e.g. using `analytics.events_flat` instead of `your_project.analytics.events_flat`).
|
|
126
152
|
This means they run cleanly out of the box in the BigQuery Web Console (which executes them in your active project workspace) or via terminal scripts.
|
|
127
153
|
|
|
128
154
|
### 5.2 Deduplicating Logs (`logs_latest`)
|
|
@@ -142,29 +168,248 @@ The `events_flat` view flattens time segments and interactions nested arrays int
|
|
|
142
168
|
|
|
143
169
|
To avoid browser copy-paste workflows and support data analyses, we provide terminal scripts inside `bigquery/scripts/`.
|
|
144
170
|
|
|
145
|
-
### 6.1 Running SQL Queries (`run_query.
|
|
146
|
-
Use `run_query.
|
|
171
|
+
### 6.1 Running SQL Queries (`run_query.js`)
|
|
172
|
+
Use `run_query.js` to run an ad hoc SQL file from the terminal and export the
|
|
173
|
+
result to JSON or CSV. Operational dashboard queries are normally run through
|
|
174
|
+
`npm run analytics:refresh`, which supplies required parameters and validates
|
|
175
|
+
the registered output schema.
|
|
147
176
|
|
|
148
177
|
```bash
|
|
149
178
|
# Export active time statistics by route to a local JSON file
|
|
150
|
-
|
|
179
|
+
node bigquery/scripts/run_query.js --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output output.json
|
|
151
180
|
|
|
152
181
|
# Export active time statistics by route to a local CSV file
|
|
153
|
-
|
|
182
|
+
node bigquery/scripts/run_query.js --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output output.csv
|
|
154
183
|
|
|
155
|
-
#
|
|
156
|
-
|
|
184
|
+
# Parameterized operational queries such as active_time_by_route.sql are run by:
|
|
185
|
+
npm run analytics:refresh
|
|
157
186
|
```
|
|
158
187
|
|
|
159
|
-
### 6.2
|
|
160
|
-
|
|
188
|
+
### 6.2 Ad Hoc Raw Entity Export (`export_entities.js`)
|
|
189
|
+
`export_entities.js` is a separate administrative utility for explicitly
|
|
190
|
+
authorized raw Firestore inspection. It is not part of snapshot generation and
|
|
191
|
+
must not be confused with the allowlisted entity derivation pipeline.
|
|
161
192
|
|
|
162
193
|
```bash
|
|
163
194
|
# Export documents directly from Firestore to a local JSON file
|
|
164
|
-
|
|
195
|
+
node bigquery/scripts/export_entities.js --credentials path/to/key.json --collection <collection-name> --output data.json
|
|
165
196
|
|
|
166
197
|
# Export documents to a local CSV file (auto-flattens nested fields)
|
|
167
|
-
|
|
198
|
+
node bigquery/scripts/export_entities.js --credentials path/to/key.json --collection <collection-name> --output data.csv
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Raw exports can contain sensitive fields and must be stored, shared, and deleted
|
|
202
|
+
according to the applicable research protocol. If credentials are omitted, the
|
|
203
|
+
scripts use Application Default Credentials.
|
|
204
|
+
|
|
205
|
+
---
|
|
206
|
+
|
|
207
|
+
## 7. Authentication Attribution
|
|
208
|
+
|
|
209
|
+
A login route begins while its visitor is anonymous. Authentication can finish
|
|
210
|
+
before the route logger opens a new signed-in segment, so a successful
|
|
211
|
+
authentication event includes the resulting user ID in its properties:
|
|
212
|
+
|
|
213
|
+
```ts
|
|
214
|
+
trackInteraction('onboarding', 'auth_result', {
|
|
215
|
+
method: 'google',
|
|
216
|
+
outcome: 'success',
|
|
217
|
+
authenticatedUserId: user.__id,
|
|
218
|
+
accessState: user.accessState,
|
|
219
|
+
});
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
Attempts, failures, and cancellations do not include this field. Cohort SQL uses
|
|
223
|
+
the route-level `user_id` when available and otherwise reads
|
|
224
|
+
`properties.authenticatedUserId`. Raw IDs remain in the protected telemetry
|
|
225
|
+
warehouse; local observations use salted keys and local dashboards receive
|
|
226
|
+
aggregates or deidentified records.
|
|
227
|
+
|
|
228
|
+
A successful authentication is not proof of account creation. It can represent
|
|
229
|
+
a returning user, guest authentication, or account linking. A true
|
|
230
|
+
account-creation cohort requires a trusted lifecycle record with an immutable
|
|
231
|
+
creation timestamp.
|
|
232
|
+
|
|
233
|
+
The activation funnel therefore admits a successful authentication only when
|
|
234
|
+
the user is not already in the `DONE` access state. A successful consent write
|
|
235
|
+
records `accessState: 'done'`; subsequent authentications by that user are
|
|
236
|
+
excluded. For historical authentication events that predate access-state
|
|
237
|
+
telemetry, a direct transition from authentication to an allowed `/home`
|
|
238
|
+
visit without any recorded interest or consent stage is treated as evidence of
|
|
239
|
+
an already-completed returning user.
|
|
240
|
+
|
|
241
|
+
Post-sign-in stages are sequential rather than independent. A user counts at
|
|
242
|
+
Consent viewed only after Interest submitted, at Consented only after Consent
|
|
243
|
+
viewed, and at Home reached only after Consented. This guarantees a
|
|
244
|
+
monotonic funnel even when telemetry is incomplete.
|
|
245
|
+
|
|
246
|
+
---
|
|
247
|
+
|
|
248
|
+
## 8. Creating a Local Snapshot
|
|
249
|
+
|
|
250
|
+
Authenticate with Google Application Default Credentials, then run:
|
|
251
|
+
|
|
252
|
+
```bash
|
|
253
|
+
npm run analytics:refresh
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
The refresh is deliberately ordered because BigQuery metrics depend on the
|
|
257
|
+
current user dimension:
|
|
258
|
+
|
|
259
|
+
1. Generates `.local/analytics/id-salt` on first use and reuses it so local
|
|
260
|
+
pseudonymous keys remain stable.
|
|
261
|
+
2. Updates the BigQuery views from repository SQL.
|
|
262
|
+
3. Incrementally synchronizes all entity-backed datasets into
|
|
263
|
+
`.local/analytics/entity-cache.sqlite`.
|
|
264
|
+
4. Fully replaces any entity table marked `uploadToBigQuery`; the generated baseline uploads
|
|
265
|
+
the current user dimension to `analytics.dim_user`.
|
|
266
|
+
5. Runs registered BigQuery datasets. Datasets marked `adminVariants` run twice:
|
|
267
|
+
once with admins included and once with admin-attributed telemetry excluded.
|
|
268
|
+
6. Inserts BigQuery results and cached entity rows into a new strict SQLite
|
|
269
|
+
database, validating every value against `analytics/datasets.js`.
|
|
270
|
+
7. Records query/entity provenance, runs `PRAGMA integrity_check`, and closes
|
|
271
|
+
the completed database.
|
|
272
|
+
8. Atomically publishes the snapshot and updates `.local/analytics/current.json`.
|
|
273
|
+
|
|
274
|
+
If a refresh fails, the previous successful snapshot remains current. Optional
|
|
275
|
+
`--format csv` and `--format json` exports contain the same allowlisted rows as
|
|
276
|
+
SQLite.
|
|
277
|
+
|
|
278
|
+
The salt and all snapshots are ignored by Git. `ANALYTICS_ID_SALT` may supply an
|
|
279
|
+
explicit salt, but sharing it also makes independently generated pseudonymous
|
|
280
|
+
keys linkable and should be treated as sensitive.
|
|
281
|
+
|
|
282
|
+
---
|
|
283
|
+
|
|
284
|
+
## 9. Incremental Entity Derivation
|
|
285
|
+
|
|
286
|
+
Project dashboards that need application entities register them in
|
|
287
|
+
`analytics/entity_sources.js` and reference them from `analytics/datasets.js`.
|
|
288
|
+
The infrastructure itself does not name application collections.
|
|
289
|
+
|
|
290
|
+
Every registered definition supplies:
|
|
291
|
+
|
|
292
|
+
- An `analysisVersion`.
|
|
293
|
+
- One or more sources with an ID, collection name, and deterministic derivation.
|
|
294
|
+
- A materializer that creates the final typed dataset rows.
|
|
295
|
+
|
|
296
|
+
The first refresh reads the registered collection. Later refreshes query
|
|
297
|
+
documents inclusively from the latest `_updatedAt` server timestamp, so new and
|
|
298
|
+
changed entities are processed while equal-timestamp documents are safely
|
|
299
|
+
replayed.
|
|
300
|
+
|
|
301
|
+
The local entity cache stores salted document keys, timestamps, versions, and
|
|
302
|
+
allowlisted derived JSON. It never stores raw IDs or raw response text.
|
|
303
|
+
|
|
304
|
+
Applications use soft deletion (`_deleted: true`) and update `_updatedAt`.
|
|
305
|
+
Incremental synchronization observes the changed document and removes its
|
|
306
|
+
cached derived record. A hard-deleted Firestore document cannot be discovered
|
|
307
|
+
after it disappears and requires a deletion journal or full reconciliation.
|
|
308
|
+
|
|
309
|
+
Force reconciliation with:
|
|
310
|
+
|
|
311
|
+
```bash
|
|
312
|
+
npm run analytics:refresh -- --rebuild-entity-cache
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
Increment `analysisVersion` whenever derivation rules change. A version or salt
|
|
316
|
+
change rebuilds that dataset's cache.
|
|
317
|
+
|
|
318
|
+
### 9.1 Generated user dimension and admin-aware metrics
|
|
319
|
+
|
|
320
|
+
The generated baseline includes a user dimension because its shared user model
|
|
321
|
+
defines `isAdmin` and `accessState`. The
|
|
322
|
+
`user-dimensions` dataset derives these fields from `users/{uid}`:
|
|
323
|
+
|
|
324
|
+
- `user_key`: salted SHA-256 of the Firebase/Firestore user ID, encoded in the
|
|
325
|
+
same lowercase hexadecimal format as BigQuery `TO_HEX(SHA256(...))`.
|
|
326
|
+
- `role`: `admin` only when `isAdmin === true`; otherwise `user`.
|
|
327
|
+
- `access_state`: allowlisted current access state.
|
|
328
|
+
- Source creation and update timestamps.
|
|
329
|
+
|
|
330
|
+
Before telemetry metrics run, the refresh uploads those rows with
|
|
331
|
+
`WRITE_TRUNCATE` to `analytics.dim_user`. Raw IDs and PII are never uploaded.
|
|
332
|
+
The operational SQL hashes protected telemetry `user_id` values with the same
|
|
333
|
+
salt and joins the resulting key to `dim_user`.
|
|
334
|
+
|
|
335
|
+
Datasets with `adminVariants: true` run twice. Their SQLite rows include an
|
|
336
|
+
`_includes_admin` discriminator: `1` means all telemetry; `0` means the query ran
|
|
337
|
+
with `@excludeAdmin = true`. The localhost API reads `includeAdmin=1|0` and
|
|
338
|
+
filters to the matching precomputed variant. Data-quality and entity datasets
|
|
339
|
+
are stored only once.
|
|
340
|
+
|
|
341
|
+
If an application removes those fields or uses a different user collection,
|
|
342
|
+
update both `analytics/entity_sources.js` and the `user-dimensions` dataset
|
|
343
|
+
contract before refreshing a snapshot.
|
|
344
|
+
|
|
345
|
+
---
|
|
346
|
+
|
|
347
|
+
## 10. Running the Workbench
|
|
348
|
+
|
|
349
|
+
```bash
|
|
350
|
+
npm run analytics
|
|
168
351
|
```
|
|
169
352
|
|
|
170
|
-
|
|
353
|
+
This starts:
|
|
354
|
+
|
|
355
|
+
- Angular workbench: `http://127.0.0.1:4320`
|
|
356
|
+
- Read-only data service: `http://127.0.0.1:4321`
|
|
357
|
+
|
|
358
|
+
Startup checks both ports before launching and reports a concise message if an
|
|
359
|
+
older analytics process is still running.
|
|
360
|
+
|
|
361
|
+
The browser cannot initiate BigQuery or Firestore refreshes. The Reload button
|
|
362
|
+
only switches the UI to a newer snapshot already created by
|
|
363
|
+
`analytics:refresh`.
|
|
364
|
+
|
|
365
|
+
The default infrastructure dashboards are:
|
|
366
|
+
|
|
367
|
+
- **Overview:** traffic, engagement, cohort conversion, snapshot recency, and
|
|
368
|
+
evidence health.
|
|
369
|
+
- **First-time journey:** a device acquisition cohort before authentication and
|
|
370
|
+
a user activation cohort afterward. Each identity counts once per stage;
|
|
371
|
+
later completion remains attributed to the original cohort.
|
|
372
|
+
- **Route engagement:** normalized routes, positive-active-time visit counts,
|
|
373
|
+
and active-time distribution summaries.
|
|
374
|
+
- **Engagement and retention:** UTC-day DAU/WAU/MAU windows and
|
|
375
|
+
Monday-cohort weekly retention. Identity prefers a known user, then the first
|
|
376
|
+
user associated with a device, then an anonymous device key.
|
|
377
|
+
- **Data quality:** missing identifiers, negative durations, malformed
|
|
378
|
+
historical properties, and event coverage.
|
|
379
|
+
|
|
380
|
+
The sidebar admin toggle does not run live SQL. It changes the API's
|
|
381
|
+
`includeAdmin` parameter and selects one of the two variants already stored in
|
|
382
|
+
the current snapshot.
|
|
383
|
+
|
|
384
|
+
---
|
|
385
|
+
|
|
386
|
+
## 11. Adding Datasets and Dashboards
|
|
387
|
+
|
|
388
|
+
`analytics/datasets.js` is the boundary for data admitted to SQLite and the
|
|
389
|
+
localhost API. Each dataset declares its source, table, grain, privacy class,
|
|
390
|
+
columns, indexes, and allowlisted filters/sorts. `entitySource` selects a local
|
|
391
|
+
derivation, `uploadToBigQuery` opts an entity dataset into a full-replacement
|
|
392
|
+
warehouse dimension, and `adminVariants` opts a parameterized query into the
|
|
393
|
+
dual include/exclude computation. The API exposes no arbitrary SQL or write
|
|
394
|
+
endpoint.
|
|
395
|
+
|
|
396
|
+
Each Angular dashboard page fetches its required datasets and renders metrics,
|
|
397
|
+
Vega-Lite charts, and tables directly. For details on the dashboard component
|
|
398
|
+
architecture, see `analytics-workbench-guide.md`.
|
|
399
|
+
|
|
400
|
+
---
|
|
401
|
+
|
|
402
|
+
## 12. Privacy and Deletion
|
|
403
|
+
|
|
404
|
+
- Do not put names, email addresses, free text, query strings, access tokens, or
|
|
405
|
+
arbitrary raw properties in SQLite.
|
|
406
|
+
- Salted identifiers reduce direct identifiability but are still linkable
|
|
407
|
+
research data.
|
|
408
|
+
- A personal-data deletion workflow must remove affected source records and
|
|
409
|
+
regenerate the current snapshot. Older local snapshots or exports containing
|
|
410
|
+
that subject's derived rows must also be removed.
|
|
411
|
+
- A refresh replaces `analytics.dim_user`, so deletion or soft-deletion from the
|
|
412
|
+
application source is reflected in the next successful upload. Superseded
|
|
413
|
+
local snapshots remain separate artifacts and still require cleanup.
|
|
414
|
+
- Access to production entity synchronization should be limited to authorized
|
|
415
|
+
researchers and developers. Credential files must never be committed.
|