@tech4good/angular-schematics 21.6.0 → 21.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +266 -1
  3. package/package.json +34 -7
  4. package/src/component/files/__name@dasherize__/__name@dasherize__.component.spec.ts.template +4 -5
  5. package/src/component/files/__name@dasherize__/__name@dasherize__.component.ts.template +1 -1
  6. package/src/entity/files/__name@dasherize__/__name@dasherize__.mock.ts.template +14 -6
  7. package/src/entity/files/__name@dasherize__/__name@dasherize__.model.ts.template +12 -6
  8. package/src/entity/files/__name@dasherize__/__name@dasherize__.store.ts.template +18 -10
  9. package/src/entity/index.js +4 -49
  10. package/src/entity/index.js.map +1 -1
  11. package/src/guard/files/__name@dasherize__.guard.spec.ts.template +0 -1
  12. package/src/ng-add/files/.github/workflows/angular-github-ci.yml.template +3 -0
  13. package/src/ng-add/files/.gitignore.template +3 -0
  14. package/src/ng-add/files/analytics/datasets.js.template +206 -0
  15. package/src/ng-add/files/analytics/entity_sources.js.template +58 -0
  16. package/src/ng-add/files/analytics/entity_sources.test.js.template +57 -0
  17. package/src/ng-add/files/analytics/lib/api_query.js.template +87 -0
  18. package/src/ng-add/files/analytics/lib/api_query.test.js.template +84 -0
  19. package/src/ng-add/files/analytics/lib/bigquery_query_compatibility.test.js.template +53 -0
  20. package/src/ng-add/files/analytics/lib/dataset_schema.js.template +53 -0
  21. package/src/ng-add/files/analytics/lib/dataset_schema.test.js.template +34 -0
  22. package/src/ng-add/files/analytics/lib/entity_sync.js.template +397 -0
  23. package/src/ng-add/files/analytics/lib/entity_sync.test.js.template +212 -0
  24. package/src/ng-add/files/analytics/lib/identifier_salt.js.template +57 -0
  25. package/src/ng-add/files/analytics/lib/identifier_salt.test.js.template +36 -0
  26. package/src/ng-add/files/analytics/lib/snapshot_files.js.template +46 -0
  27. package/src/ng-add/files/analytics/lib/snapshot_files.test.js.template +67 -0
  28. package/src/ng-add/files/analytics/lib/sqlite.js.template +92 -0
  29. package/src/ng-add/files/analytics/lib/upload_entities.js.template +81 -0
  30. package/src/ng-add/files/analytics/lib/upload_entities.test.js.template +86 -0
  31. package/src/ng-add/files/analytics/proxy.conf.json.template +7 -0
  32. package/src/ng-add/files/analytics/scripts/refresh_snapshot.js.template +518 -0
  33. package/src/ng-add/files/analytics/scripts/start.js.template +88 -0
  34. package/src/ng-add/files/analytics/server/server.js.template +212 -0
  35. package/src/ng-add/files/angular.json.template +80 -1
  36. package/src/ng-add/files/bigquery/queries/active_time_by_route.sql.template +27 -8
  37. package/src/ng-add/files/bigquery/queries/active_user_windows.sql.template +67 -0
  38. package/src/ng-add/files/bigquery/queries/dashboard_route_visits.sql.template +47 -0
  39. package/src/ng-add/files/bigquery/queries/data_quality.sql.template +323 -0
  40. package/src/ng-add/files/bigquery/queries/onboarding_funnel.sql.template +512 -0
  41. package/src/ng-add/files/bigquery/queries/onboarding_stage_observations.sql.template +134 -0
  42. package/src/ng-add/files/bigquery/queries/weekly_retention_cohorts.sql.template +90 -0
  43. package/src/ng-add/files/bigquery/scripts/export_entities.js.template +120 -0
  44. package/src/ng-add/files/bigquery/scripts/init_views.js.template +57 -0
  45. package/src/ng-add/files/bigquery/scripts/run_query.js.template +141 -0
  46. package/src/ng-add/files/bigquery/views/events_flat.sql.template +27 -12
  47. package/src/ng-add/files/bigquery/views/logs_latest.sql.template +16 -3
  48. package/src/ng-add/files/docs-infra/analytics-guide.md.template +270 -25
  49. package/src/ng-add/files/docs-infra/analytics-workbench-guide.md.template +307 -0
  50. package/src/ng-add/files/docs-infra/authentication-architecture.md.template +54 -24
  51. package/src/ng-add/files/docs-infra/email-infrastructure.md.template +43 -0
  52. package/src/ng-add/files/docs-infra/firestore-index-checker.md.template +87 -0
  53. package/src/ng-add/files/docs-infra/infrastructure-guide.md.template +108 -29
  54. package/src/ng-add/files/docs-infra/load-system-architecture.md.template +15 -1
  55. package/src/ng-add/files/docs-infra/querying-strategy.md.template +14 -0
  56. package/src/ng-add/files/docs-infra/stream-system-architecture.md.template +21 -7
  57. package/src/ng-add/files/eslint.config.js.template +22 -3
  58. package/src/ng-add/files/functions/package.json.template +1 -0
  59. package/src/ng-add/files/functions/src/email.ts.template +23 -0
  60. package/src/ng-add/files/jest.config.ts.template +3 -2
  61. package/src/ng-add/files/package.json.template +39 -16
  62. package/src/ng-add/files/projects/analytics/src/app/analytics-api.service.ts.template +130 -0
  63. package/src/ng-add/files/projects/analytics/src/app/app.component.html.template +77 -0
  64. package/src/ng-add/files/projects/analytics/src/app/app.component.scss.template +240 -0
  65. package/src/ng-add/files/projects/analytics/src/app/app.component.ts.template +96 -0
  66. package/src/ng-add/files/projects/analytics/src/app/app.routes.ts.template +11 -0
  67. package/src/ng-add/files/projects/analytics/src/app/dashboard-pages.spec.ts.template +151 -0
  68. package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.html.template +39 -0
  69. package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.scss.template +1 -0
  70. package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.ts.template +95 -0
  71. package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.html.template +47 -0
  72. package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.scss.template +1 -0
  73. package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.ts.template +120 -0
  74. package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.html.template +40 -0
  75. package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.scss.template +1 -0
  76. package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.ts.template +98 -0
  77. package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.html.template +35 -0
  78. package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.scss.template +1 -0
  79. package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.ts.template +84 -0
  80. package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.html.template +42 -0
  81. package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.scss.template +1 -0
  82. package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.ts.template +110 -0
  83. package/src/ng-add/files/projects/analytics/src/app/shared/analytics-utils.ts.template +43 -0
  84. package/src/ng-add/files/projects/analytics/src/app/shared/chart-theme.ts.template +42 -0
  85. package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-layout.scss.template +239 -0
  86. package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.html.template +76 -0
  87. package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.scss.template +128 -0
  88. package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.spec.ts.template +31 -0
  89. package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.ts.template +153 -0
  90. package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.html.template +16 -0
  91. package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.scss.template +63 -0
  92. package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.ts.template +52 -0
  93. package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.html.template +5 -0
  94. package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.scss.template +56 -0
  95. package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.spec.ts.template +30 -0
  96. package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.ts.template +15 -0
  97. package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.html.template +9 -0
  98. package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.scss.template +43 -0
  99. package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.ts.template +62 -0
  100. package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.html.template +9 -0
  101. package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.scss.template +33 -0
  102. package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.ts.template +82 -0
  103. package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.html.template +15 -0
  104. package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.scss.template +45 -0
  105. package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.ts.template +55 -0
  106. package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.html.template +18 -0
  107. package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.scss.template +45 -0
  108. package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.ts.template +15 -0
  109. package/src/ng-add/files/projects/analytics/src/app/vega-chart/vega-chart.component.ts.template +121 -0
  110. package/src/ng-add/files/projects/analytics/src/index.html.template +19 -0
  111. package/src/ng-add/files/projects/analytics/src/main.ts.template +16 -0
  112. package/src/ng-add/files/projects/analytics/src/styles.scss.template +28 -0
  113. package/src/ng-add/files/projects/analytics/tsconfig.app.json.template +13 -0
  114. package/src/ng-add/files/projects/analytics/tsconfig.spec.json.template +15 -0
  115. package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.spec.ts.template +4 -5
  116. package/src/ng-add/files/src/app/admin/admin-panel/admin-settings/admin-settings.component.spec.ts.template +4 -5
  117. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.spec.ts.template +11 -12
  118. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.ts.template +7 -7
  119. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.spec.ts.template +8 -10
  120. package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.ts.template +3 -2
  121. package/src/ng-add/files/src/app/app.component.html.template +1 -1
  122. package/src/ng-add/files/src/app/app.component.scss.template +1 -1
  123. package/src/ng-add/files/src/app/app.component.spec.ts.template +4 -5
  124. package/src/ng-add/files/src/app/app.component.ts.template +1 -2
  125. package/src/ng-add/files/src/app/app.config.ts.template +21 -15
  126. package/src/ng-add/files/src/app/auth-routing.spec.ts.template +4 -5
  127. package/src/ng-add/files/src/app/core/analytics/time-analytics.service.spec.ts.template +46 -29
  128. package/src/ng-add/files/src/app/core/analytics/time-analytics.service.ts.template +27 -18
  129. package/src/ng-add/files/src/app/core/auth/auth.providers.spec.ts.template +112 -0
  130. package/src/ng-add/files/src/app/core/auth/auth.providers.ts.template +48 -0
  131. package/src/ng-add/files/src/app/core/auth/auth.service.ts.template +9 -23
  132. package/src/ng-add/files/src/app/core/auth/guards/require-admin.guard.spec.ts.template +3 -14
  133. package/src/ng-add/files/src/app/core/firebase/data.providers.prod.ts.template +21 -0
  134. package/src/ng-add/files/src/app/core/firebase/data.providers.ts.template +13 -0
  135. package/src/ng-add/files/src/app/core/local-storage/session.service.spec.ts.template +43 -0
  136. package/src/ng-add/files/src/app/core/local-storage/session.service.ts.template +7 -0
  137. package/src/ng-add/files/src/app/core/store/app-data.spec.ts.template +76 -0
  138. package/src/ng-add/files/src/app/core/store/auth/auth.mock.ts.template +19 -67
  139. package/src/ng-add/files/src/app/core/store/auth/auth.store.spec.ts.template +102 -12
  140. package/src/ng-add/files/src/app/core/store/auth/auth.store.ts.template +32 -237
  141. package/src/ng-add/files/src/app/core/store/user/user.mock.ts.template +17 -12
  142. package/src/ng-add/files/src/app/core/store/user/user.model.ts.template +6 -7
  143. package/src/ng-add/files/src/app/core/store/user/user.store.ts.template +14 -10
  144. package/src/ng-add/files/src/app/core/store/user-context/user-context.mock.ts.template +20 -8
  145. package/src/ng-add/files/src/app/core/store/user-context/user-context.model.ts.template +5 -6
  146. package/src/ng-add/files/src/app/core/store/user-context/user-context.store.ts.template +14 -10
  147. package/src/ng-add/files/src/app/core/store/user-private/user-private.mock.ts.template +20 -7
  148. package/src/ng-add/files/src/app/core/store/user-private/user-private.model.ts.template +5 -6
  149. package/src/ng-add/files/src/app/core/store/user-private/user-private.store.ts.template +14 -10
  150. package/src/ng-add/files/src/app/core/testing/app-mocks.ts.template +42 -0
  151. package/src/ng-add/files/src/app/first-time/consent/consent.component.html.template +2 -1
  152. package/src/ng-add/files/src/app/first-time/consent/consent.component.scss.template +7 -3
  153. package/src/ng-add/files/src/app/first-time/consent/consent.component.spec.ts.template +31 -11
  154. package/src/ng-add/files/src/app/first-time/consent/consent.component.ts.template +16 -1
  155. package/src/ng-add/files/src/app/first-time/early-access/early-access.component.spec.ts.template +15 -15
  156. package/src/ng-add/files/src/app/first-time/early-access/early-access.component.ts.template +11 -3
  157. package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.html.template +2 -1
  158. package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.scss.template +7 -3
  159. package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.spec.ts.template +20 -6
  160. package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.ts.template +2 -0
  161. package/src/ng-add/files/src/app/first-time/early-access/waitlist-view/waitlist-view.component.spec.ts.template +4 -5
  162. package/src/ng-add/files/src/app/first-time/email-link/email-link.component.spec.ts.template +17 -1
  163. package/src/ng-add/files/src/app/first-time/email-link/email-link.component.ts.template +15 -0
  164. package/src/ng-add/files/src/app/first-time/first-time-routing.spec.ts.template +4 -5
  165. package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.ts.template +15 -0
  166. package/src/ng-add/files/src/app/first-time/landing/landing.component.spec.ts.template +2 -0
  167. package/src/ng-add/files/src/app/first-time/login/login.component.spec.ts.template +21 -3
  168. package/src/ng-add/files/src/app/first-time/login/login.component.ts.template +26 -0
  169. package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.spec.ts.template +4 -2
  170. package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.ts.template +1 -1
  171. package/src/ng-add/files/src/app/main/account-profile/account-profile.component.spec.ts.template +2 -2
  172. package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.spec.ts.template +2 -2
  173. package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.spec.ts.template +4 -4
  174. package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.spec.ts.template +2 -2
  175. package/src/ng-add/files/src/app/main/home/home.component.spec.ts.template +4 -5
  176. package/src/ng-add/files/src/app/main/home/home.component.ts.template +1 -1
  177. package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.spec.ts.template +4 -4
  178. package/src/ng-add/files/src/app/shared/navbar/navbar.component.spec.ts.template +2 -2
  179. package/src/ng-add/files/tsconfig.spec.json.template +3 -1
  180. package/src/ng-add/index.js +7 -0
  181. package/src/ng-add/index.js.map +1 -1
  182. package/src/shared/entity-registry.d.ts +8 -0
  183. package/src/shared/entity-registry.js +28 -0
  184. package/src/shared/entity-registry.js.map +1 -0
  185. package/.agents/skills/publish-npm/SKILL.md +0 -37
  186. package/.agents/skills/publish-npm/eval.yaml +0 -32
  187. package/.agents/skills/publish-npm/fixtures/package-v21.json.template +0 -8
  188. package/.husky/commit-msg +0 -1
  189. package/.husky/pre-commit +0 -1
  190. package/.t4g/agent-memory.db +0 -0
  191. package/src/component/index_spec.d.ts +0 -1
  192. package/src/component/index_spec.js +0 -68
  193. package/src/component/index_spec.js.map +0 -1
  194. package/src/entity/index_spec.d.ts +0 -1
  195. package/src/entity/index_spec.js +0 -153
  196. package/src/entity/index_spec.js.map +0 -1
  197. package/src/guard/index_spec.d.ts +0 -1
  198. package/src/guard/index_spec.js +0 -64
  199. package/src/guard/index_spec.js.map +0 -1
  200. package/src/ng-add/files/bigquery/scripts/export_entities.py.template +0 -143
  201. package/src/ng-add/files/bigquery/scripts/run_query.py.template +0 -167
  202. package/src/ng-add/files/src/app/core/firebase/cached-listeners.service.ts.template +0 -500
  203. package/src/ng-add/files/src/app/core/firebase/database.service.ts.template +0 -73
  204. package/src/ng-add/files/src/app/core/firebase/email-link-context.service.spec.ts.template +0 -59
  205. package/src/ng-add/files/src/app/core/firebase/email-link-context.service.ts.template +0 -147
  206. package/src/ng-add/files/src/app/core/firebase/firebase.mock.service.ts.template +0 -453
  207. package/src/ng-add/files/src/app/core/firebase/firebase.service.spec.ts.template +0 -116
  208. package/src/ng-add/files/src/app/core/firebase/firebase.service.ts.template +0 -639
  209. package/src/ng-add/files/src/app/core/firebase/simulate-load.spec.ts.template +0 -104
  210. package/src/ng-add/files/src/app/core/firebase/simulate-stream.spec.ts.template +0 -127
  211. package/src/ng-add/files/src/app/core/firebase/simulation.mock.ts.template +0 -38
  212. package/src/ng-add/files/src/app/core/firebase/stream-multiplexer.ts.template +0 -445
  213. package/src/ng-add/files/src/app/core/logging/logging.service.spec.ts.template +0 -118
  214. package/src/ng-add/files/src/app/core/logging/logging.service.ts.template +0 -168
  215. package/src/ng-add/files/src/app/core/store/app.model.ts.template +0 -21
  216. package/src/ng-add/files/src/app/core/store/app.store.spec.ts.template +0 -920
  217. package/src/ng-add/files/src/app/core/store/app.store.ts.template +0 -1320
  218. package/src/ng-add/files/src/app/core/store/batch-write.firebase.service.ts.template +0 -58
  219. package/src/ng-add/files/src/app/core/store/batch-write.mock.service.ts.template +0 -105
  220. package/src/ng-add/files/src/app/core/store/batch-write.service.ts.template +0 -126
  221. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.html.template +0 -9
  222. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.scss.template +0 -46
  223. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.spec.ts.template +0 -69
  224. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.ts.template +0 -39
  225. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.spec.ts.template +0 -122
  226. package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.ts.template +0 -88
  227. package/src/ng-add/index_spec.d.ts +0 -1
  228. package/src/ng-add/index_spec.js +0 -108
  229. package/src/ng-add/index_spec.js.map +0 -1
@@ -0,0 +1,90 @@
1
+ -- Metric contract: weekly retention from each identity's first active week.
2
+ -- Purpose: For every Monday-starting UTC cohort week, report the share of
3
+ -- the original cohort active in week 0, week 1, and each observable later week.
4
+ -- Grain: One output row per cohort_week x weeks_since_first_active cell.
5
+ -- Identity: event user_id, otherwise the first user ever observed on that
6
+ -- device, otherwise anonymous:<device_id>. Session IDs are never identities.
7
+ -- Activity: at least one positive-active-time route visit in the week. DISTINCT
8
+ -- identity/week pairs prevent repeated visits from inflating retention.
9
+ -- Admin semantics: @excludeAdmin removes identified admins and pre-auth activity
10
+ -- from devices later tied to an admin; truly never-identified devices remain.
11
+ -- Interpretation: week 0 is always 100% by construction and describes cohort
12
+ -- membership, not a return. Later cells measure observed return behavior.
13
+ -- SQL approach: resolve identity/week pairs, assign MIN(active_week) as cohort,
14
+ -- generate a complete elapsed-week grid through the latest observed week, count
15
+ -- active identities, and SAFE_DIVIDE by the fixed cohort size.
16
+
17
+ WITH known_user_by_device AS (
18
+ -- Use eventual device attribution consistently with active_user_windows.sql.
19
+ SELECT
20
+ device_id,
21
+ ARRAY_AGG(user_id IGNORE NULLS ORDER BY timestamp LIMIT 1)[SAFE_OFFSET(0)] AS user_id
22
+ FROM `analytics.events_flat`
23
+ WHERE device_id IS NOT NULL
24
+ GROUP BY device_id
25
+ ),
26
+ active_weeks AS (
27
+ -- One row per identity/week is the sufficient statistic for weekly retention.
28
+ SELECT DISTINCT
29
+ COALESCE(event.user_id, known.user_id, CONCAT('anonymous:', event.device_id)) AS identity_key,
30
+ DATE_TRUNC(DATE(TIMESTAMP_MILLIS(event.timestamp), 'America/Los_Angeles'), WEEK(MONDAY)) AS active_week
31
+ FROM `analytics.events_flat` AS event
32
+ LEFT JOIN known_user_by_device AS known USING (device_id)
33
+ WHERE event.event_type = 'route_visit'
34
+ AND event.device_id IS NOT NULL
35
+ AND event.timestamp IS NOT NULL
36
+ AND event.total_active_ms > 0
37
+ AND (
38
+ NOT @excludeAdmin
39
+ OR COALESCE(event.user_id, known.user_id) IS NULL
40
+ OR TO_HEX(SHA256(CONCAT(@idSalt, ':', COALESCE(event.user_id, known.user_id), ':', @idSalt))) NOT IN (
41
+ SELECT user_key FROM `analytics.dim_user` WHERE role = 'admin'
42
+ )
43
+ )
44
+ ),
45
+ cohorted AS (
46
+ SELECT
47
+ identity_key,
48
+ active_week,
49
+ MIN(active_week) OVER (PARTITION BY identity_key) AS cohort_week
50
+ FROM active_weeks
51
+ ),
52
+ cohort_sizes AS (
53
+ SELECT cohort_week, COUNT(DISTINCT identity_key) AS cohort_size
54
+ FROM cohorted
55
+ WHERE active_week = cohort_week
56
+ GROUP BY cohort_week
57
+ ),
58
+ max_week AS (
59
+ SELECT MAX(active_week) AS max_active_week FROM active_weeks
60
+ ),
61
+ cohort_weeks_grid AS (
62
+ -- Generate missing elapsed weeks so zero-retention cells are explicit.
63
+ SELECT
64
+ c.cohort_week,
65
+ weeks_since_first_active,
66
+ c.cohort_size
67
+ FROM cohort_sizes AS c
68
+ CROSS JOIN max_week AS m
69
+ CROSS JOIN UNNEST(GENERATE_ARRAY(0, DATE_DIFF(m.max_active_week, c.cohort_week, WEEK))) AS weeks_since_first_active
70
+ ),
71
+ retention AS (
72
+ SELECT
73
+ cohort_week,
74
+ DATE_DIFF(active_week, cohort_week, WEEK) AS weeks_since_first_active,
75
+ COUNT(DISTINCT identity_key) AS active_users
76
+ FROM cohorted
77
+ GROUP BY cohort_week, weeks_since_first_active
78
+ )
79
+ SELECT
80
+ CONCAT(CAST(grid.cohort_week AS STRING), ':', CAST(grid.weeks_since_first_active AS STRING)) AS retention_key,
81
+ CAST(grid.cohort_week AS STRING) AS cohort_week,
82
+ grid.weeks_since_first_active,
83
+ COALESCE(retention.active_users, 0) AS active_users,
84
+ grid.cohort_size,
85
+ COALESCE(SAFE_DIVIDE(retention.active_users, grid.cohort_size), 0.0) AS retention_rate
86
+ FROM cohort_weeks_grid AS grid
87
+ LEFT JOIN retention
88
+ ON grid.cohort_week = retention.cohort_week
89
+ AND grid.weeks_since_first_active = retention.weeks_since_first_active
90
+ ORDER BY cohort_week, weeks_since_first_active;
@@ -0,0 +1,120 @@
1
+ /**
2
+ * Firestore Entity Exporter (`export_entities.js`)
3
+ *
4
+ * Exports a Firestore collection to a local JSON or CSV file.
5
+ *
6
+ * Usage:
7
+ * node bigquery/scripts/export_entities.js --collection userResponses --output data.json
8
+ * node bigquery/scripts/export_entities.js --collection userResponses --output data.csv --credentials key.json
9
+ */
10
+
11
+ import admin from 'firebase-admin';
12
+ import fs from 'fs';
13
+ import path from 'path';
14
+
15
+ /** Extracts named command line flag argument */
16
+ function getArg(flag) {
17
+ const index = process.argv.indexOf(flag);
18
+ return index !== -1 ? process.argv[index + 1] : null;
19
+ }
20
+
21
+ /** Main CLI execution function */
22
+ async function main() {
23
+ const collectionName = getArg('--collection');
24
+ const outputPath = getArg('--output');
25
+ const keyPath = getArg('--credentials') || process.env.GOOGLE_APPLICATION_CREDENTIALS;
26
+ const projectId = getArg('--project') || '<%= dasherize(project) %>';
27
+
28
+ if (!collectionName || !outputPath) {
29
+ console.error('Usage: node bigquery/scripts/export_entities.js --collection <name> --output <out.json|csv> [--credentials <key.json>] [--project <id>]');
30
+ process.exit(1);
31
+ }
32
+
33
+ // Authenticate Firebase Admin SDK
34
+ if (!admin.apps.length) {
35
+ try {
36
+ if (keyPath && fs.existsSync(keyPath)) {
37
+ const serviceAccount = JSON.parse(fs.readFileSync(keyPath, 'utf8'));
38
+ admin.initializeApp({
39
+ credential: admin.credential.cert(serviceAccount),
40
+ projectId: serviceAccount.project_id || projectId,
41
+ });
42
+ } else {
43
+ admin.initializeApp({
44
+ credential: admin.credential.applicationDefault(),
45
+ projectId,
46
+ });
47
+ }
48
+ } catch (error) {
49
+ console.error('Authentication Error: Could not initialize Firebase Admin SDK.');
50
+ console.error('Details:', error.message);
51
+ process.exit(1);
52
+ }
53
+ }
54
+
55
+ const db = admin.firestore();
56
+ console.log(`Fetching documents from collection '${collectionName}'...`);
57
+
58
+ try {
59
+ // Fetch Firestore document collection stream
60
+ const snapshot = await db.collection(collectionName).get();
61
+ console.log(`Retrieved ${snapshot.size} documents.`);
62
+
63
+ if (snapshot.empty) {
64
+ console.log('No documents found. Exiting.');
65
+ return;
66
+ }
67
+
68
+ const flatData = [];
69
+ const headersSet = new Set(['__id']);
70
+
71
+ // Normalize timestamps and build tabular header set
72
+ snapshot.forEach(doc => {
73
+ const data = doc.data();
74
+ const row = { __id: doc.id };
75
+
76
+ for (const [k, v] of Object.entries(data)) {
77
+ headersSet.add(k);
78
+ if (v && typeof v === 'object' && typeof v.toDate === 'function') {
79
+ row[k] = v.toDate().toISOString();
80
+ } else {
81
+ row[k] = v;
82
+ }
83
+ }
84
+ flatData.push(row);
85
+ });
86
+
87
+ const headers = Array.from(headersSet);
88
+ const absOutputPath = path.resolve(process.cwd(), outputPath);
89
+ const ext = path.extname(outputPath).toLowerCase();
90
+
91
+ // Export to JSON or CSV
92
+ if (ext === '.json') {
93
+ fs.writeFileSync(absOutputPath, JSON.stringify(flatData, null, 2), 'utf8');
94
+ console.log(`Successfully exported data to JSON: ${absOutputPath}`);
95
+ } else if (ext === '.csv') {
96
+ const csvLines = [headers.join(',')];
97
+
98
+ for (const row of flatData) {
99
+ const line = headers.map(h => {
100
+ const val = row[h];
101
+ if (val === null || val === undefined) return '';
102
+ const str = typeof val === 'object' ? JSON.stringify(val) : String(val);
103
+ return `"${str.replace(/"/g, '""')}"`;
104
+ }).join(',');
105
+ csvLines.push(line);
106
+ }
107
+
108
+ fs.writeFileSync(absOutputPath, csvLines.join('\n'), 'utf8');
109
+ console.log(`Successfully exported data to CSV: ${absOutputPath}`);
110
+ } else {
111
+ console.error(`Error: Unknown output format '${ext}'. Must be .json or .csv.`);
112
+ process.exit(1);
113
+ }
114
+ } catch (error) {
115
+ console.error('Export Error:', error.message);
116
+ process.exit(1);
117
+ }
118
+ }
119
+
120
+ main();
@@ -0,0 +1,57 @@
1
+ /**
2
+ * Deploys BigQuery analytics views from the SQL files in bigquery/views/.
3
+ *
4
+ * Usage:
5
+ * node bigquery/scripts/init_views.js [--project <gcp-project-id>]
6
+ */
7
+
8
+ import { BigQuery } from '@google-cloud/bigquery';
9
+ import fs from 'fs';
10
+ import path from 'path';
11
+ import { fileURLToPath } from 'url';
12
+
13
+ const __filename = fileURLToPath(import.meta.url);
14
+ const __dirname = path.dirname(__filename);
15
+
16
+ async function main() {
17
+ const projectIndex = process.argv.indexOf('--project');
18
+ const projectId = projectIndex !== -1 ? process.argv[projectIndex + 1] : '<%= dasherize(project) %>-logs';
19
+
20
+ const repoRoot = path.resolve(__dirname, '..', '..');
21
+ const logsLatestPath = path.join(repoRoot, 'bigquery', 'views', 'logs_latest.sql');
22
+ const eventsFlatPath = path.join(repoRoot, 'bigquery', 'views', 'events_flat.sql');
23
+
24
+ if (!fs.existsSync(logsLatestPath)) {
25
+ console.error(`Error: View SQL file not found at ${logsLatestPath}`);
26
+ process.exit(1);
27
+ }
28
+ if (!fs.existsSync(eventsFlatPath)) {
29
+ console.error(`Error: View SQL file not found at ${eventsFlatPath}`);
30
+ process.exit(1);
31
+ }
32
+
33
+ const logsLatestSql = fs.readFileSync(logsLatestPath, 'utf8');
34
+ const eventsFlatSql = fs.readFileSync(eventsFlatPath, 'utf8');
35
+
36
+ console.log(`Initializing BigQuery analytics views on GCP project '${projectId}'...`);
37
+ const bigquery = new BigQuery({ projectId });
38
+
39
+ try {
40
+ console.log('Creating/updating view `analytics.logs_latest`...');
41
+ await bigquery.query({ query: logsLatestSql });
42
+ console.log('✓ View `analytics.logs_latest` created successfully.');
43
+
44
+ console.log('Creating/updating view `analytics.events_flat`...');
45
+ await bigquery.query({ query: eventsFlatSql });
46
+ console.log('✓ View `analytics.events_flat` created successfully.');
47
+
48
+ console.log('\nSuccessfully initialized all BigQuery analytics views!');
49
+ } catch (error) {
50
+ console.error('\nBigQuery Execution Error:', error.message);
51
+ console.error('Please verify that you are authenticated with Google Cloud:');
52
+ console.error(' gcloud auth application-default login');
53
+ process.exit(1);
54
+ }
55
+ }
56
+
57
+ main();
@@ -0,0 +1,141 @@
1
+ /**
2
+ * BigQuery Query Runner Script (`run_query.js`)
3
+ *
4
+ * Runs a BigQuery SQL file and exports results to JSON, CSV, or SQLite.
5
+ *
6
+ * Usage:
7
+ * node bigquery/scripts/run_query.js --query bigquery/queries/active_time_by_route.sql --output output.json [--project <log-project-id>]
8
+ * node bigquery/scripts/run_query.js --query bigquery/queries/active_time_by_route.sql --output stats.db --table active_time [--project <log-project-id>]
9
+ */
10
+
11
+ import { BigQuery } from '@google-cloud/bigquery';
12
+ import fs from 'fs';
13
+ import path from 'path';
14
+ import sqlite3 from 'sqlite3';
15
+ import { fileURLToPath } from 'url';
16
+
17
+ const __filename = fileURLToPath(import.meta.url);
18
+ const __dirname = path.dirname(__filename);
19
+
20
+ /** Extracts named command line flag argument */
21
+ function getArg(flag) {
22
+ const index = process.argv.indexOf(flag);
23
+ return index !== -1 ? process.argv[index + 1] : null;
24
+ }
25
+
26
+ /** Main query execution entrypoint */
27
+ async function main() {
28
+ const queryPath = getArg('--query');
29
+ const outputPath = getArg('--output');
30
+ const projectId = getArg('--project') || '<%= dasherize(project) %>-logs';
31
+ const tableName = getArg('--table');
32
+ const keyPath = getArg('--credentials');
33
+
34
+ if (!queryPath || !outputPath) {
35
+ console.error('Usage: node bigquery/scripts/run_query.js --query <file.sql> --output <out.json|csv|db> [--project <id>] [--table <name>] [--credentials <key.json>]');
36
+ process.exit(1);
37
+ }
38
+
39
+ const absQueryPath = path.resolve(process.cwd(), queryPath);
40
+ if (!fs.existsSync(absQueryPath)) {
41
+ console.error(`Error: Query file not found at ${absQueryPath}`);
42
+ process.exit(1);
43
+ }
44
+
45
+ const outputExt = path.extname(outputPath).toLowerCase();
46
+ const isSqlite = ['.db', '.sqlite', '.sqlite3'].includes(outputExt);
47
+
48
+ if (isSqlite && !tableName) {
49
+ console.error('Error: --table argument is required when exporting to a SQLite database.');
50
+ process.exit(1);
51
+ }
52
+
53
+ // Read SQL query string from file
54
+ const queryString = fs.readFileSync(absQueryPath, 'utf8');
55
+
56
+ const options = { projectId };
57
+ if (keyPath) {
58
+ options.keyFilename = keyPath;
59
+ }
60
+
61
+ const bigquery = new BigQuery(options);
62
+
63
+ console.log(`Executing query '${queryPath}' in GCP project '${projectId}'...`);
64
+ try {
65
+ // Execute BigQuery job asynchronously
66
+ const [rows] = await bigquery.query({ query: queryString });
67
+ console.log(`Query completed successfully. Fetched ${rows.length} rows.`);
68
+
69
+ if (!rows.length) {
70
+ console.log('Query returned 0 rows. Exiting.');
71
+ return;
72
+ }
73
+
74
+ const absOutputPath = path.resolve(process.cwd(), outputPath);
75
+
76
+ // Write results based on target file format
77
+ if (outputExt === '.json') {
78
+ fs.writeFileSync(absOutputPath, JSON.stringify(rows, null, 2), 'utf8');
79
+ console.log(`Successfully exported ${rows.length} rows to JSON: ${absOutputPath}`);
80
+ } else if (outputExt === '.csv') {
81
+ const headers = Object.keys(rows[0]);
82
+ const csvLines = [headers.join(',')];
83
+
84
+ for (const row of rows) {
85
+ const line = headers.map(h => {
86
+ const val = row[h];
87
+ if (val === null || val === undefined) return '';
88
+ const str = typeof val === 'object' ? JSON.stringify(val) : String(val);
89
+ return `"${str.replace(/"/g, '""')}"`;
90
+ }).join(',');
91
+ csvLines.push(line);
92
+ }
93
+
94
+ fs.writeFileSync(absOutputPath, csvLines.join('\n'), 'utf8');
95
+ console.log(`Successfully exported ${rows.length} rows to CSV: ${absOutputPath}`);
96
+ } else if (isSqlite) {
97
+ const db = new sqlite3.Database(absOutputPath);
98
+ const headers = Object.keys(rows[0]);
99
+
100
+ // Infer column types for dynamic SQLite table creation
101
+ const columnsDef = headers.map(h => {
102
+ const val = rows[0][h];
103
+ let colType = 'TEXT';
104
+ if (typeof val === 'number') {
105
+ colType = Number.isInteger(val) ? 'INTEGER' : 'REAL';
106
+ }
107
+ return `\`${h}\` ${colType}`;
108
+ }).join(', ');
109
+
110
+ db.serialize(() => {
111
+ db.run(`CREATE TABLE IF NOT EXISTS \`${tableName}\` (${columnsDef})`);
112
+
113
+ const placeholders = headers.map(() => '?').join(', ');
114
+ const insertSql = `INSERT INTO \`${tableName}\` (${headers.map(h => `\`${h}\``).join(', ')}) VALUES (${placeholders})`;
115
+
116
+ const stmt = db.prepare(insertSql);
117
+ for (const row of rows) {
118
+ const rowData = headers.map(h => {
119
+ const val = row[h];
120
+ if (val === null || val === undefined) return null;
121
+ return typeof val === 'object' ? JSON.stringify(val) : val;
122
+ });
123
+ stmt.run(rowData);
124
+ }
125
+ stmt.finalize();
126
+ });
127
+
128
+ db.close(() => {
129
+ console.log(`Successfully imported ${rows.length} rows into SQLite table '${tableName}': ${absOutputPath}`);
130
+ });
131
+ } else {
132
+ console.error(`Error: Unknown output format '${outputExt}'. Must be .json, .csv, or .db/.sqlite.`);
133
+ process.exit(1);
134
+ }
135
+ } catch (error) {
136
+ console.error('Query Execution Error:', error.message);
137
+ process.exit(1);
138
+ }
139
+ }
140
+
141
+ main();
@@ -1,34 +1,46 @@
1
- -- View: events_flat
2
- -- Description: Flattens the nested time segments and interactions arrays in the latest logs
3
- -- into a single clean event-stream format with one row per event.
4
- -- Preserves the raw chronological time segments array for advanced engagement pattern analysis.
1
+ -- View contract: canonical event-level telemetry relation.
2
+ -- Purpose: Convert each latest Firestore route-log document into one route_visit
3
+ -- row plus zero or more interaction rows with a shared column schema.
4
+ -- Grain: route_visit rows represent finalized route/state segments; interaction
5
+ -- rows represent one [timestamp, category, action, properties] tuple.
6
+ -- Identity: raw protected warehouse IDs are retained here for attribution. Any
7
+ -- dataset leaving BigQuery must aggregate them or replace them with salted keys.
8
+ -- Timing: route timestamp is startTime; active/rest totals sum the chronological
9
+ -- time_segments array. Interaction timestamp comes from tuple element 0.
10
+ -- SQL approach: SELECT route rows from analytics.logs_latest, UNION ALL a second
11
+ -- SELECT that UNNESTs interactions, and fill non-applicable columns with typed
12
+ -- NULLs so both event types share one stable schema.
5
13
 
6
14
  CREATE OR REPLACE VIEW `analytics.events_flat` AS
7
- -- 1. Route visits (emitted when a route is closed/navigated away from)
15
+ -- Route visits: one row per latest route/state segment.
8
16
  SELECT
9
17
  document_id,
10
18
  __userId AS user_id,
11
19
  __sessionId AS session_id,
12
20
  __deviceId AS device_id,
13
21
  route,
22
+ state,
14
23
  _createdAt,
15
24
  _updatedAt,
16
25
  'route_visit' AS event_type,
17
26
  startTime AS timestamp,
27
+ -- Interaction fields set to NULL for route visit rows
18
28
  CAST(NULL AS STRING) AS category,
19
29
  CAST(NULL AS STRING) AS action,
20
30
  CAST(NULL AS STRING) AS properties,
21
- -- Preserves the chronological pattern of active/rest intervals (JSON array)
31
+ -- Preserve raw JSON time segments array for custom engagement pattern analysis
22
32
  time_segments,
33
+ -- Sum total active milliseconds from JSON array elements where type = 'active'
23
34
  COALESCE(
24
- (SELECT SUM(CAST(JSON_VALUE(t, '$.total') AS INT64))
25
- FROM UNNEST(JSON_QUERY_ARRAY(time_segments)) t
35
+ (SELECT SUM(CAST(JSON_VALUE(t, '$.total') AS INT64))
36
+ FROM UNNEST(JSON_QUERY_ARRAY(time_segments)) t
26
37
  WHERE JSON_VALUE(t, '$.type') = 'active'),
27
38
  0
28
39
  ) AS total_active_ms,
40
+ -- Sum total rest/idle milliseconds from JSON array elements where type = 'rest'
29
41
  COALESCE(
30
- (SELECT SUM(CAST(JSON_VALUE(t, '$.total') AS INT64))
31
- FROM UNNEST(JSON_QUERY_ARRAY(time_segments)) t
42
+ (SELECT SUM(CAST(JSON_VALUE(t, '$.total') AS INT64))
43
+ FROM UNNEST(JSON_QUERY_ARRAY(time_segments)) t
32
44
  WHERE JSON_VALUE(t, '$.type') = 'rest'),
33
45
  0
34
46
  ) AS total_rest_ms
@@ -37,21 +49,24 @@ FROM
37
49
 
38
50
  UNION ALL
39
51
 
40
- -- 2. Interactions (discrete click, focus, and submission events)
52
+ -- Interactions: UNNEST preserves every tuple; UNION ALL intentionally does not
53
+ -- deduplicate repeated instrumentation events.
41
54
  SELECT
42
55
  document_id,
43
56
  __userId AS user_id,
44
57
  __sessionId AS session_id,
45
58
  __deviceId AS device_id,
46
59
  route,
60
+ state,
47
61
  _createdAt,
48
62
  _updatedAt,
49
63
  'interaction' AS event_type,
64
+ -- Extract tuple attributes from interactions array: [timestamp, category, action, properties]
50
65
  CAST(JSON_VALUE(ix, '$[0]') AS INT64) AS timestamp,
51
66
  JSON_VALUE(ix, '$[1]') AS category,
52
67
  JSON_VALUE(ix, '$[2]') AS action,
53
68
  JSON_VALUE(ix, '$[3]') AS properties,
54
- -- Null for interactions
69
+ -- Timing fields set to NULL for interaction rows
55
70
  CAST(NULL AS STRING) AS time_segments,
56
71
  CAST(NULL AS INT64) AS total_active_ms,
57
72
  CAST(NULL AS INT64) AS total_rest_ms
@@ -1,22 +1,35 @@
1
- -- View: logs_latest
2
- -- Description: Extracts the latest version of each log document from the raw changelog table,
3
- -- deduplicating documents that have been updated (e.g., via sendBeacon writes).
1
+ -- View contract: one current row per Firestore telemetry document.
2
+ -- Purpose: Reconstruct the latest document state from the append-only Firestore
3
+ -- BigQuery Export changelog before event-level flattening.
4
+ -- Grain: One non-deleted row per document_id.
5
+ -- SQL approach: ROW_NUMBER each document's changelog records by BigQuery sync
6
+ -- timestamp descending, retain rn = 1, and discard latest DELETE operations.
7
+ -- Field handling: JSON_VALUE extracts scalar identifiers/routes/timestamps;
8
+ -- JSON_QUERY preserves state, time, and interaction structures for events_flat.
9
+ -- Important distinction: bq_sync_timestamp orders warehouse versions, while
10
+ -- startTime/_createdAt/_updatedAt are client event/document timestamps.
4
11
 
5
12
  CREATE OR REPLACE VIEW `analytics.logs_latest` AS
6
13
  SELECT
7
14
  document_id,
8
15
  timestamp AS bq_sync_timestamp,
9
16
  operation,
17
+ -- Extract user, session, and device metadata from JSON payload
10
18
  JSON_VALUE(data, '$.__userId') AS __userId,
11
19
  JSON_VALUE(data, '$.__sessionId') AS __sessionId,
12
20
  JSON_VALUE(data, '$.__deviceId') AS __deviceId,
13
21
  JSON_VALUE(data, '$.route') AS route,
22
+ -- Extract optional application state context (for example, an active item ID)
23
+ JSON_QUERY(data, '$.state') AS state,
14
24
  CAST(JSON_VALUE(data, '$.startTime') AS INT64) AS startTime,
25
+ -- Raw arrays for telemetry breakdown
15
26
  JSON_QUERY(data, '$.time') AS time_segments,
16
27
  JSON_QUERY(data, '$.interactions') AS interactions,
28
+ -- Client-side timestamps
17
29
  CAST(JSON_VALUE(data, '$._createdAt') AS INT64) AS _createdAt,
18
30
  CAST(JSON_VALUE(data, '$._updatedAt') AS INT64) AS _updatedAt
19
31
  FROM (
32
+ -- Rank append-only CDC versions so updates do not double-count a log.
20
33
  SELECT
21
34
  document_id,
22
35
  timestamp,