@tech4good/angular-schematics 21.6.0 → 21.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +266 -1
- package/package.json +34 -7
- package/src/component/files/__name@dasherize__/__name@dasherize__.component.spec.ts.template +4 -5
- package/src/component/files/__name@dasherize__/__name@dasherize__.component.ts.template +1 -1
- package/src/entity/files/__name@dasherize__/__name@dasherize__.mock.ts.template +14 -6
- package/src/entity/files/__name@dasherize__/__name@dasherize__.model.ts.template +12 -6
- package/src/entity/files/__name@dasherize__/__name@dasherize__.store.ts.template +18 -10
- package/src/entity/index.js +4 -49
- package/src/entity/index.js.map +1 -1
- package/src/guard/files/__name@dasherize__.guard.spec.ts.template +0 -1
- package/src/ng-add/files/.github/workflows/angular-github-ci.yml.template +3 -0
- package/src/ng-add/files/.gitignore.template +3 -0
- package/src/ng-add/files/analytics/datasets.js.template +206 -0
- package/src/ng-add/files/analytics/entity_sources.js.template +58 -0
- package/src/ng-add/files/analytics/entity_sources.test.js.template +57 -0
- package/src/ng-add/files/analytics/lib/api_query.js.template +87 -0
- package/src/ng-add/files/analytics/lib/api_query.test.js.template +84 -0
- package/src/ng-add/files/analytics/lib/bigquery_query_compatibility.test.js.template +53 -0
- package/src/ng-add/files/analytics/lib/dataset_schema.js.template +53 -0
- package/src/ng-add/files/analytics/lib/dataset_schema.test.js.template +34 -0
- package/src/ng-add/files/analytics/lib/entity_sync.js.template +397 -0
- package/src/ng-add/files/analytics/lib/entity_sync.test.js.template +212 -0
- package/src/ng-add/files/analytics/lib/identifier_salt.js.template +57 -0
- package/src/ng-add/files/analytics/lib/identifier_salt.test.js.template +36 -0
- package/src/ng-add/files/analytics/lib/snapshot_files.js.template +46 -0
- package/src/ng-add/files/analytics/lib/snapshot_files.test.js.template +67 -0
- package/src/ng-add/files/analytics/lib/sqlite.js.template +92 -0
- package/src/ng-add/files/analytics/lib/upload_entities.js.template +81 -0
- package/src/ng-add/files/analytics/lib/upload_entities.test.js.template +86 -0
- package/src/ng-add/files/analytics/proxy.conf.json.template +7 -0
- package/src/ng-add/files/analytics/scripts/refresh_snapshot.js.template +518 -0
- package/src/ng-add/files/analytics/scripts/start.js.template +88 -0
- package/src/ng-add/files/analytics/server/server.js.template +212 -0
- package/src/ng-add/files/angular.json.template +80 -1
- package/src/ng-add/files/bigquery/queries/active_time_by_route.sql.template +27 -8
- package/src/ng-add/files/bigquery/queries/active_user_windows.sql.template +67 -0
- package/src/ng-add/files/bigquery/queries/dashboard_route_visits.sql.template +47 -0
- package/src/ng-add/files/bigquery/queries/data_quality.sql.template +323 -0
- package/src/ng-add/files/bigquery/queries/onboarding_funnel.sql.template +512 -0
- package/src/ng-add/files/bigquery/queries/onboarding_stage_observations.sql.template +134 -0
- package/src/ng-add/files/bigquery/queries/weekly_retention_cohorts.sql.template +90 -0
- package/src/ng-add/files/bigquery/scripts/export_entities.js.template +120 -0
- package/src/ng-add/files/bigquery/scripts/init_views.js.template +57 -0
- package/src/ng-add/files/bigquery/scripts/run_query.js.template +141 -0
- package/src/ng-add/files/bigquery/views/events_flat.sql.template +27 -12
- package/src/ng-add/files/bigquery/views/logs_latest.sql.template +16 -3
- package/src/ng-add/files/docs-infra/analytics-guide.md.template +270 -25
- package/src/ng-add/files/docs-infra/analytics-workbench-guide.md.template +307 -0
- package/src/ng-add/files/docs-infra/authentication-architecture.md.template +54 -24
- package/src/ng-add/files/docs-infra/email-infrastructure.md.template +43 -0
- package/src/ng-add/files/docs-infra/firestore-index-checker.md.template +87 -0
- package/src/ng-add/files/docs-infra/infrastructure-guide.md.template +108 -29
- package/src/ng-add/files/docs-infra/load-system-architecture.md.template +15 -1
- package/src/ng-add/files/docs-infra/querying-strategy.md.template +14 -0
- package/src/ng-add/files/docs-infra/stream-system-architecture.md.template +21 -7
- package/src/ng-add/files/eslint.config.js.template +22 -3
- package/src/ng-add/files/functions/package.json.template +1 -0
- package/src/ng-add/files/functions/src/email.ts.template +23 -0
- package/src/ng-add/files/jest.config.ts.template +3 -2
- package/src/ng-add/files/package.json.template +39 -16
- package/src/ng-add/files/projects/analytics/src/app/analytics-api.service.ts.template +130 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.html.template +77 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.scss.template +240 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.ts.template +96 -0
- package/src/ng-add/files/projects/analytics/src/app/app.routes.ts.template +11 -0
- package/src/ng-add/files/projects/analytics/src/app/dashboard-pages.spec.ts.template +151 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.html.template +39 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.ts.template +95 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.html.template +47 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.ts.template +120 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.html.template +40 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.ts.template +98 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.html.template +35 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.ts.template +84 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.html.template +42 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.ts.template +110 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/analytics-utils.ts.template +43 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/chart-theme.ts.template +42 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-layout.scss.template +239 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.html.template +76 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.scss.template +128 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.spec.ts.template +31 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.ts.template +153 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.html.template +16 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.scss.template +63 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.ts.template +52 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.html.template +5 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.scss.template +56 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.spec.ts.template +30 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.ts.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.html.template +9 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.scss.template +43 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.ts.template +62 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.html.template +9 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.scss.template +33 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.ts.template +82 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.html.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.scss.template +45 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.ts.template +55 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.html.template +18 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.scss.template +45 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.ts.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/vega-chart/vega-chart.component.ts.template +121 -0
- package/src/ng-add/files/projects/analytics/src/index.html.template +19 -0
- package/src/ng-add/files/projects/analytics/src/main.ts.template +16 -0
- package/src/ng-add/files/projects/analytics/src/styles.scss.template +28 -0
- package/src/ng-add/files/projects/analytics/tsconfig.app.json.template +13 -0
- package/src/ng-add/files/projects/analytics/tsconfig.spec.json.template +15 -0
- package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/admin/admin-panel/admin-settings/admin-settings.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.spec.ts.template +11 -12
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.ts.template +7 -7
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.spec.ts.template +8 -10
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.ts.template +3 -2
- package/src/ng-add/files/src/app/app.component.html.template +1 -1
- package/src/ng-add/files/src/app/app.component.scss.template +1 -1
- package/src/ng-add/files/src/app/app.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/app.component.ts.template +1 -2
- package/src/ng-add/files/src/app/app.config.ts.template +21 -15
- package/src/ng-add/files/src/app/auth-routing.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.spec.ts.template +46 -29
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.ts.template +27 -18
- package/src/ng-add/files/src/app/core/auth/auth.providers.spec.ts.template +112 -0
- package/src/ng-add/files/src/app/core/auth/auth.providers.ts.template +48 -0
- package/src/ng-add/files/src/app/core/auth/auth.service.ts.template +9 -23
- package/src/ng-add/files/src/app/core/auth/guards/require-admin.guard.spec.ts.template +3 -14
- package/src/ng-add/files/src/app/core/firebase/data.providers.prod.ts.template +21 -0
- package/src/ng-add/files/src/app/core/firebase/data.providers.ts.template +13 -0
- package/src/ng-add/files/src/app/core/local-storage/session.service.spec.ts.template +43 -0
- package/src/ng-add/files/src/app/core/local-storage/session.service.ts.template +7 -0
- package/src/ng-add/files/src/app/core/store/app-data.spec.ts.template +76 -0
- package/src/ng-add/files/src/app/core/store/auth/auth.mock.ts.template +19 -67
- package/src/ng-add/files/src/app/core/store/auth/auth.store.spec.ts.template +102 -12
- package/src/ng-add/files/src/app/core/store/auth/auth.store.ts.template +32 -237
- package/src/ng-add/files/src/app/core/store/user/user.mock.ts.template +17 -12
- package/src/ng-add/files/src/app/core/store/user/user.model.ts.template +6 -7
- package/src/ng-add/files/src/app/core/store/user/user.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/store/user-context/user-context.mock.ts.template +20 -8
- package/src/ng-add/files/src/app/core/store/user-context/user-context.model.ts.template +5 -6
- package/src/ng-add/files/src/app/core/store/user-context/user-context.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/store/user-private/user-private.mock.ts.template +20 -7
- package/src/ng-add/files/src/app/core/store/user-private/user-private.model.ts.template +5 -6
- package/src/ng-add/files/src/app/core/store/user-private/user-private.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/testing/app-mocks.ts.template +42 -0
- package/src/ng-add/files/src/app/first-time/consent/consent.component.html.template +2 -1
- package/src/ng-add/files/src/app/first-time/consent/consent.component.scss.template +7 -3
- package/src/ng-add/files/src/app/first-time/consent/consent.component.spec.ts.template +31 -11
- package/src/ng-add/files/src/app/first-time/consent/consent.component.ts.template +16 -1
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.spec.ts.template +15 -15
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.ts.template +11 -3
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.html.template +2 -1
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.scss.template +7 -3
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.spec.ts.template +20 -6
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.ts.template +2 -0
- package/src/ng-add/files/src/app/first-time/early-access/waitlist-view/waitlist-view.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.spec.ts.template +17 -1
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.ts.template +15 -0
- package/src/ng-add/files/src/app/first-time/first-time-routing.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.ts.template +15 -0
- package/src/ng-add/files/src/app/first-time/landing/landing.component.spec.ts.template +2 -0
- package/src/ng-add/files/src/app/first-time/login/login.component.spec.ts.template +21 -3
- package/src/ng-add/files/src/app/first-time/login/login.component.ts.template +26 -0
- package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.spec.ts.template +4 -2
- package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.ts.template +1 -1
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.spec.ts.template +4 -4
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/main/home/home.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/main/home/home.component.ts.template +1 -1
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.spec.ts.template +4 -4
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.spec.ts.template +2 -2
- package/src/ng-add/files/tsconfig.spec.json.template +3 -1
- package/src/ng-add/index.js +7 -0
- package/src/ng-add/index.js.map +1 -1
- package/src/shared/entity-registry.d.ts +8 -0
- package/src/shared/entity-registry.js +28 -0
- package/src/shared/entity-registry.js.map +1 -0
- package/.agents/skills/publish-npm/SKILL.md +0 -37
- package/.agents/skills/publish-npm/eval.yaml +0 -32
- package/.agents/skills/publish-npm/fixtures/package-v21.json.template +0 -8
- package/.husky/commit-msg +0 -1
- package/.husky/pre-commit +0 -1
- package/.t4g/agent-memory.db +0 -0
- package/src/component/index_spec.d.ts +0 -1
- package/src/component/index_spec.js +0 -68
- package/src/component/index_spec.js.map +0 -1
- package/src/entity/index_spec.d.ts +0 -1
- package/src/entity/index_spec.js +0 -153
- package/src/entity/index_spec.js.map +0 -1
- package/src/guard/index_spec.d.ts +0 -1
- package/src/guard/index_spec.js +0 -64
- package/src/guard/index_spec.js.map +0 -1
- package/src/ng-add/files/bigquery/scripts/export_entities.py.template +0 -143
- package/src/ng-add/files/bigquery/scripts/run_query.py.template +0 -167
- package/src/ng-add/files/src/app/core/firebase/cached-listeners.service.ts.template +0 -500
- package/src/ng-add/files/src/app/core/firebase/database.service.ts.template +0 -73
- package/src/ng-add/files/src/app/core/firebase/email-link-context.service.spec.ts.template +0 -59
- package/src/ng-add/files/src/app/core/firebase/email-link-context.service.ts.template +0 -147
- package/src/ng-add/files/src/app/core/firebase/firebase.mock.service.ts.template +0 -453
- package/src/ng-add/files/src/app/core/firebase/firebase.service.spec.ts.template +0 -116
- package/src/ng-add/files/src/app/core/firebase/firebase.service.ts.template +0 -639
- package/src/ng-add/files/src/app/core/firebase/simulate-load.spec.ts.template +0 -104
- package/src/ng-add/files/src/app/core/firebase/simulate-stream.spec.ts.template +0 -127
- package/src/ng-add/files/src/app/core/firebase/simulation.mock.ts.template +0 -38
- package/src/ng-add/files/src/app/core/firebase/stream-multiplexer.ts.template +0 -445
- package/src/ng-add/files/src/app/core/logging/logging.service.spec.ts.template +0 -118
- package/src/ng-add/files/src/app/core/logging/logging.service.ts.template +0 -168
- package/src/ng-add/files/src/app/core/store/app.model.ts.template +0 -21
- package/src/ng-add/files/src/app/core/store/app.store.spec.ts.template +0 -920
- package/src/ng-add/files/src/app/core/store/app.store.ts.template +0 -1320
- package/src/ng-add/files/src/app/core/store/batch-write.firebase.service.ts.template +0 -58
- package/src/ng-add/files/src/app/core/store/batch-write.mock.service.ts.template +0 -105
- package/src/ng-add/files/src/app/core/store/batch-write.service.ts.template +0 -126
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.html.template +0 -9
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.scss.template +0 -46
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.spec.ts.template +0 -69
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.ts.template +0 -39
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.spec.ts.template +0 -122
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.ts.template +0 -88
- package/src/ng-add/index_spec.d.ts +0 -1
- package/src/ng-add/index_spec.js +0 -108
- package/src/ng-add/index_spec.js.map +0 -1
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Incremental Firestore-to-SQLite Entity Synchronizer
|
|
3
|
+
*
|
|
4
|
+
* Manages incremental sync of derived entities from live Firestore collections
|
|
5
|
+
* into a local SQLite cache.
|
|
6
|
+
*
|
|
7
|
+
* Architecture Notes:
|
|
8
|
+
* - Pseudonymization: Document keys are hashed with `idSalt` to prevent raw IDs from being stored unhashed.
|
|
9
|
+
* - Cache Invalidation: Automatic if `analysisVersion`, salt key, or target collection name changes.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { createHash } from 'node:crypto';
|
|
13
|
+
import { FieldPath, Timestamp } from 'firebase-admin/firestore';
|
|
14
|
+
import { all, exec, run } from './sqlite.js';
|
|
15
|
+
|
|
16
|
+
const DEFAULT_PAGE_SIZE = 500;
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
function quoteIdentifier(identifier) {
|
|
20
|
+
if (!/^[a-z][a-z0-9_]*$/i.test(identifier)) {
|
|
21
|
+
throw new Error(`Unsafe SQLite identifier: ${identifier}`);
|
|
22
|
+
}
|
|
23
|
+
return `"${identifier}"`;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
function timestampParts(value) {
|
|
28
|
+
if (!value || typeof value !== 'object') return null;
|
|
29
|
+
const seconds = Number(value.seconds ?? value._seconds);
|
|
30
|
+
const nanoseconds = Number(value.nanoseconds ?? value._nanoseconds ?? 0);
|
|
31
|
+
if (!Number.isInteger(seconds) || !Number.isInteger(nanoseconds)) return null;
|
|
32
|
+
return {
|
|
33
|
+
seconds,
|
|
34
|
+
nanoseconds,
|
|
35
|
+
milliseconds: (seconds * 1000) + Math.floor(nanoseconds / 1_000_000),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
function compareTimestamps(left, right) {
|
|
41
|
+
if (!left) return right ? -1 : 0;
|
|
42
|
+
if (!right) return 1;
|
|
43
|
+
return left.seconds === right.seconds ?
|
|
44
|
+
left.nanoseconds - right.nanoseconds :
|
|
45
|
+
left.seconds - right.seconds;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Generates an irreversible salted SHA256 key for cache primary keys */
|
|
49
|
+
function cacheDocumentKey(idSalt, source, documentId) {
|
|
50
|
+
return createHash('sha256')
|
|
51
|
+
.update(`${idSalt}:${source.collection}:${documentId}:${idSalt}`)
|
|
52
|
+
.digest('hex')
|
|
53
|
+
.toUpperCase();
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Generates SHA256 hash of idSalt to verify salt consistency between syncs */
|
|
57
|
+
function saltFingerprint(idSalt) {
|
|
58
|
+
return createHash('sha256').update(idSalt).digest('hex');
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
export async function initializeEntityCache(db) {
|
|
63
|
+
await exec(db, `
|
|
64
|
+
CREATE TABLE IF NOT EXISTS entity_sync_state (
|
|
65
|
+
dataset_id TEXT NOT NULL,
|
|
66
|
+
source_id TEXT NOT NULL,
|
|
67
|
+
collection_name TEXT NOT NULL,
|
|
68
|
+
analysis_version INTEGER NOT NULL,
|
|
69
|
+
salt_fingerprint TEXT NOT NULL,
|
|
70
|
+
initialized INTEGER NOT NULL,
|
|
71
|
+
cursor_seconds INTEGER,
|
|
72
|
+
cursor_nanoseconds INTEGER,
|
|
73
|
+
PRIMARY KEY (dataset_id, source_id)
|
|
74
|
+
) STRICT;
|
|
75
|
+
CREATE TABLE IF NOT EXISTS entity_derived_cache (
|
|
76
|
+
dataset_id TEXT NOT NULL,
|
|
77
|
+
source_id TEXT NOT NULL,
|
|
78
|
+
document_key TEXT NOT NULL,
|
|
79
|
+
analysis_version INTEGER NOT NULL,
|
|
80
|
+
source_updated_at_ms INTEGER NOT NULL,
|
|
81
|
+
derived_json TEXT NOT NULL,
|
|
82
|
+
PRIMARY KEY (dataset_id, source_id, document_key)
|
|
83
|
+
) STRICT;
|
|
84
|
+
CREATE INDEX IF NOT EXISTS idx_entity_derived_cache_dataset
|
|
85
|
+
ON entity_derived_cache (dataset_id, source_id);
|
|
86
|
+
`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Resets cache rows for a dataset when schema, salt, or analysis version changes */
|
|
90
|
+
async function resetDatasetCache(db, datasetId) {
|
|
91
|
+
await exec(db, 'BEGIN IMMEDIATE');
|
|
92
|
+
try {
|
|
93
|
+
await run(db, 'DELETE FROM entity_derived_cache WHERE dataset_id = ?', [datasetId]);
|
|
94
|
+
await run(db, 'DELETE FROM entity_sync_state WHERE dataset_id = ?', [datasetId]);
|
|
95
|
+
await exec(db, 'COMMIT');
|
|
96
|
+
} catch (error) {
|
|
97
|
+
await exec(db, 'ROLLBACK');
|
|
98
|
+
throw error;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
async function applyDocuments(
|
|
104
|
+
db,
|
|
105
|
+
datasetId,
|
|
106
|
+
definition,
|
|
107
|
+
source,
|
|
108
|
+
documents,
|
|
109
|
+
idSalt,
|
|
110
|
+
) {
|
|
111
|
+
const upsertSql = `
|
|
112
|
+
INSERT INTO entity_derived_cache (
|
|
113
|
+
dataset_id, source_id, document_key, analysis_version,
|
|
114
|
+
source_updated_at_ms, derived_json
|
|
115
|
+
) VALUES (?, ?, ?, ?, ?, ?)
|
|
116
|
+
ON CONFLICT (dataset_id, source_id, document_key) DO UPDATE SET
|
|
117
|
+
analysis_version = excluded.analysis_version,
|
|
118
|
+
source_updated_at_ms = excluded.source_updated_at_ms,
|
|
119
|
+
derived_json = excluded.derived_json
|
|
120
|
+
`;
|
|
121
|
+
let latestTimestamp = null;
|
|
122
|
+
|
|
123
|
+
await exec(db, 'BEGIN IMMEDIATE');
|
|
124
|
+
try {
|
|
125
|
+
for (const document of documents) {
|
|
126
|
+
const data = document.data();
|
|
127
|
+
const updatedAt = timestampParts(data._updatedAt);
|
|
128
|
+
if (updatedAt && compareTimestamps(updatedAt, latestTimestamp) > 0) {
|
|
129
|
+
latestTimestamp = updatedAt;
|
|
130
|
+
}
|
|
131
|
+
const documentKey = cacheDocumentKey(idSalt, source, document.id);
|
|
132
|
+
if (data._deleted === true) {
|
|
133
|
+
await run(
|
|
134
|
+
db,
|
|
135
|
+
`DELETE FROM entity_derived_cache
|
|
136
|
+
WHERE dataset_id = ? AND source_id = ? AND document_key = ?`,
|
|
137
|
+
[datasetId, source.id, documentKey],
|
|
138
|
+
);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const createdAt = timestampParts(data._createdAt);
|
|
143
|
+
const derived = source.derive(data, {
|
|
144
|
+
documentId: document.id,
|
|
145
|
+
idSalt,
|
|
146
|
+
sourceUpdatedAtMs: updatedAt?.milliseconds ?? 0,
|
|
147
|
+
sourceCreatedAtMs: createdAt?.milliseconds ?? null,
|
|
148
|
+
});
|
|
149
|
+
if (!derived) {
|
|
150
|
+
await run(
|
|
151
|
+
db,
|
|
152
|
+
`DELETE FROM entity_derived_cache
|
|
153
|
+
WHERE dataset_id = ? AND source_id = ? AND document_key = ?`,
|
|
154
|
+
[datasetId, source.id, documentKey],
|
|
155
|
+
);
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
await run(db, upsertSql, [
|
|
160
|
+
datasetId,
|
|
161
|
+
source.id,
|
|
162
|
+
documentKey,
|
|
163
|
+
definition.analysisVersion,
|
|
164
|
+
updatedAt?.milliseconds ?? 0,
|
|
165
|
+
JSON.stringify(derived),
|
|
166
|
+
]);
|
|
167
|
+
}
|
|
168
|
+
await exec(db, 'COMMIT');
|
|
169
|
+
} catch (error) {
|
|
170
|
+
await exec(db, 'ROLLBACK');
|
|
171
|
+
throw error;
|
|
172
|
+
}
|
|
173
|
+
return latestTimestamp;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
async function saveState(db, datasetId, definition, source, fingerprint, cursor) {
|
|
178
|
+
await run(
|
|
179
|
+
db,
|
|
180
|
+
`INSERT INTO entity_sync_state (
|
|
181
|
+
dataset_id, source_id, collection_name, analysis_version,
|
|
182
|
+
salt_fingerprint, initialized, cursor_seconds, cursor_nanoseconds
|
|
183
|
+
) VALUES (?, ?, ?, ?, ?, 1, ?, ?)
|
|
184
|
+
ON CONFLICT (dataset_id, source_id) DO UPDATE SET
|
|
185
|
+
collection_name = excluded.collection_name,
|
|
186
|
+
analysis_version = excluded.analysis_version,
|
|
187
|
+
salt_fingerprint = excluded.salt_fingerprint,
|
|
188
|
+
initialized = excluded.initialized,
|
|
189
|
+
cursor_seconds = excluded.cursor_seconds,
|
|
190
|
+
cursor_nanoseconds = excluded.cursor_nanoseconds`,
|
|
191
|
+
[
|
|
192
|
+
datasetId,
|
|
193
|
+
source.id,
|
|
194
|
+
source.collection,
|
|
195
|
+
definition.analysisVersion,
|
|
196
|
+
fingerprint,
|
|
197
|
+
cursor?.seconds ?? null,
|
|
198
|
+
cursor?.nanoseconds ?? null,
|
|
199
|
+
],
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
async function bootstrapSource(
|
|
205
|
+
firestore,
|
|
206
|
+
db,
|
|
207
|
+
datasetId,
|
|
208
|
+
definition,
|
|
209
|
+
source,
|
|
210
|
+
idSalt,
|
|
211
|
+
fingerprint,
|
|
212
|
+
pageSize,
|
|
213
|
+
) {
|
|
214
|
+
let lastDocument = null;
|
|
215
|
+
let latestTimestamp = null;
|
|
216
|
+
|
|
217
|
+
while (true) {
|
|
218
|
+
let query = firestore
|
|
219
|
+
.collection(source.collection)
|
|
220
|
+
.orderBy(FieldPath.documentId())
|
|
221
|
+
.limit(pageSize);
|
|
222
|
+
if (lastDocument) query = query.startAfter(lastDocument);
|
|
223
|
+
const snapshot = await query.get();
|
|
224
|
+
if (snapshot.empty) break;
|
|
225
|
+
|
|
226
|
+
const pageTimestamp = await applyDocuments(
|
|
227
|
+
db,
|
|
228
|
+
datasetId,
|
|
229
|
+
definition,
|
|
230
|
+
source,
|
|
231
|
+
snapshot.docs,
|
|
232
|
+
idSalt,
|
|
233
|
+
);
|
|
234
|
+
if (compareTimestamps(pageTimestamp, latestTimestamp) > 0) {
|
|
235
|
+
latestTimestamp = pageTimestamp;
|
|
236
|
+
}
|
|
237
|
+
lastDocument = snapshot.docs.at(-1);
|
|
238
|
+
if (snapshot.size < pageSize) break;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
await saveState(
|
|
242
|
+
db,
|
|
243
|
+
datasetId,
|
|
244
|
+
definition,
|
|
245
|
+
source,
|
|
246
|
+
fingerprint,
|
|
247
|
+
latestTimestamp,
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
async function updateSource(
|
|
253
|
+
firestore,
|
|
254
|
+
db,
|
|
255
|
+
datasetId,
|
|
256
|
+
definition,
|
|
257
|
+
source,
|
|
258
|
+
state,
|
|
259
|
+
idSalt,
|
|
260
|
+
fingerprint,
|
|
261
|
+
pageSize,
|
|
262
|
+
) {
|
|
263
|
+
const savedCursor = state.cursor_seconds === null ?
|
|
264
|
+
null :
|
|
265
|
+
{
|
|
266
|
+
seconds: state.cursor_seconds,
|
|
267
|
+
nanoseconds: state.cursor_nanoseconds,
|
|
268
|
+
};
|
|
269
|
+
let pageCursor = null;
|
|
270
|
+
let latestTimestamp = savedCursor;
|
|
271
|
+
|
|
272
|
+
while (true) {
|
|
273
|
+
let query = firestore
|
|
274
|
+
.collection(source.collection)
|
|
275
|
+
.orderBy('_updatedAt')
|
|
276
|
+
.orderBy(FieldPath.documentId());
|
|
277
|
+
if (pageCursor) {
|
|
278
|
+
query = query.startAfter(
|
|
279
|
+
new Timestamp(pageCursor.seconds, pageCursor.nanoseconds),
|
|
280
|
+
pageCursor.documentId,
|
|
281
|
+
);
|
|
282
|
+
} else if (savedCursor) {
|
|
283
|
+
query = query.startAt(
|
|
284
|
+
new Timestamp(savedCursor.seconds, savedCursor.nanoseconds),
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
const snapshot = await query.limit(pageSize).get();
|
|
288
|
+
if (snapshot.empty) break;
|
|
289
|
+
|
|
290
|
+
const pageTimestamp = await applyDocuments(
|
|
291
|
+
db,
|
|
292
|
+
datasetId,
|
|
293
|
+
definition,
|
|
294
|
+
source,
|
|
295
|
+
snapshot.docs,
|
|
296
|
+
idSalt,
|
|
297
|
+
);
|
|
298
|
+
if (compareTimestamps(pageTimestamp, latestTimestamp) > 0) {
|
|
299
|
+
latestTimestamp = pageTimestamp;
|
|
300
|
+
}
|
|
301
|
+
const lastDocument = snapshot.docs.at(-1);
|
|
302
|
+
const lastTimestamp = timestampParts(lastDocument.data()._updatedAt);
|
|
303
|
+
if (!lastTimestamp) {
|
|
304
|
+
throw new Error(
|
|
305
|
+
`A document in ${source.collection} is missing a valid _updatedAt timestamp`,
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
pageCursor = { ...lastTimestamp, documentId: lastDocument.id };
|
|
309
|
+
if (snapshot.size < pageSize) break;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
await saveState(
|
|
313
|
+
db,
|
|
314
|
+
datasetId,
|
|
315
|
+
definition,
|
|
316
|
+
source,
|
|
317
|
+
fingerprint,
|
|
318
|
+
latestTimestamp,
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
export async function syncEntityDataset({
|
|
324
|
+
firestore,
|
|
325
|
+
cacheDb,
|
|
326
|
+
datasetId,
|
|
327
|
+
definition,
|
|
328
|
+
idSalt,
|
|
329
|
+
pageSize = DEFAULT_PAGE_SIZE,
|
|
330
|
+
forceRebuild = false,
|
|
331
|
+
}) {
|
|
332
|
+
await initializeEntityCache(cacheDb);
|
|
333
|
+
const fingerprint = saltFingerprint(idSalt);
|
|
334
|
+
const states = await all(
|
|
335
|
+
cacheDb,
|
|
336
|
+
'SELECT * FROM entity_sync_state WHERE dataset_id = ?',
|
|
337
|
+
[datasetId],
|
|
338
|
+
);
|
|
339
|
+
const statesBySource = new Map(states.map((state) => [state.source_id, state]));
|
|
340
|
+
const requiresRebuild =
|
|
341
|
+
forceRebuild ||
|
|
342
|
+
states.length !== definition.sources.length ||
|
|
343
|
+
definition.sources.some((source) => {
|
|
344
|
+
const state = statesBySource.get(source.id);
|
|
345
|
+
return !state ||
|
|
346
|
+
state.collection_name !== source.collection ||
|
|
347
|
+
state.analysis_version !== definition.analysisVersion ||
|
|
348
|
+
state.salt_fingerprint !== fingerprint ||
|
|
349
|
+
state.initialized !== 1;
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
if (requiresRebuild) {
|
|
353
|
+
await resetDatasetCache(cacheDb, datasetId);
|
|
354
|
+
statesBySource.clear();
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
for (const source of definition.sources) {
|
|
358
|
+
const state = statesBySource.get(source.id);
|
|
359
|
+
if (!state) {
|
|
360
|
+
await bootstrapSource(
|
|
361
|
+
firestore,
|
|
362
|
+
cacheDb,
|
|
363
|
+
datasetId,
|
|
364
|
+
definition,
|
|
365
|
+
source,
|
|
366
|
+
idSalt,
|
|
367
|
+
fingerprint,
|
|
368
|
+
pageSize,
|
|
369
|
+
);
|
|
370
|
+
} else {
|
|
371
|
+
await updateSource(
|
|
372
|
+
firestore,
|
|
373
|
+
cacheDb,
|
|
374
|
+
datasetId,
|
|
375
|
+
definition,
|
|
376
|
+
source,
|
|
377
|
+
state,
|
|
378
|
+
idSalt,
|
|
379
|
+
fingerprint,
|
|
380
|
+
pageSize,
|
|
381
|
+
);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
const cachedRows = await all(
|
|
386
|
+
cacheDb,
|
|
387
|
+
`SELECT source_id, derived_json
|
|
388
|
+
FROM ${quoteIdentifier('entity_derived_cache')}
|
|
389
|
+
WHERE dataset_id = ?
|
|
390
|
+
ORDER BY source_id, document_key`,
|
|
391
|
+
[datasetId],
|
|
392
|
+
);
|
|
393
|
+
return definition.materialize(cachedRows.map((row) => ({
|
|
394
|
+
sourceId: row.source_id,
|
|
395
|
+
derived: JSON.parse(row.derived_json),
|
|
396
|
+
})));
|
|
397
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import assert from 'node:assert/strict';
|
|
5
|
+
import { describe, it } from 'node:test';
|
|
6
|
+
import { Timestamp } from 'firebase-admin/firestore';
|
|
7
|
+
import sqlite3 from 'sqlite3';
|
|
8
|
+
import { all, closeDatabase, openDatabase } from './sqlite.js';
|
|
9
|
+
import { syncEntityDataset } from './entity_sync.js';
|
|
10
|
+
|
|
11
|
+
function compareTimestamp(left, right) {
|
|
12
|
+
if (left.seconds !== right.seconds) return left.seconds - right.seconds;
|
|
13
|
+
return left.nanoseconds - right.nanoseconds;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
class FakeDocument {
|
|
17
|
+
constructor(id, value) {
|
|
18
|
+
this.id = id;
|
|
19
|
+
this.value = value;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
data() {
|
|
23
|
+
return this.value;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
class FakeQuery {
|
|
28
|
+
constructor(documents) {
|
|
29
|
+
this.documents = documents;
|
|
30
|
+
this.updatedAtOrder = false;
|
|
31
|
+
this.startAtTimestamp = null;
|
|
32
|
+
this.startAfterValues = null;
|
|
33
|
+
this.pageSize = Number.POSITIVE_INFINITY;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
orderBy(field) {
|
|
37
|
+
if (field === '_updatedAt') this.updatedAtOrder = true;
|
|
38
|
+
return this;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
startAt(timestamp) {
|
|
42
|
+
this.startAtTimestamp = timestamp;
|
|
43
|
+
return this;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
startAfter(...values) {
|
|
47
|
+
this.startAfterValues = values;
|
|
48
|
+
return this;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
limit(value) {
|
|
52
|
+
this.pageSize = value;
|
|
53
|
+
return this;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async get() {
|
|
57
|
+
let documents = this.documents.map(
|
|
58
|
+
({ id, data }) => new FakeDocument(id, data),
|
|
59
|
+
);
|
|
60
|
+
if (this.updatedAtOrder) {
|
|
61
|
+
documents.sort((left, right) => {
|
|
62
|
+
const timestampOrder = compareTimestamp(
|
|
63
|
+
left.data()._updatedAt,
|
|
64
|
+
right.data()._updatedAt,
|
|
65
|
+
);
|
|
66
|
+
return timestampOrder || left.id.localeCompare(right.id);
|
|
67
|
+
});
|
|
68
|
+
if (this.startAtTimestamp) {
|
|
69
|
+
documents = documents.filter(
|
|
70
|
+
(document) =>
|
|
71
|
+
compareTimestamp(document.data()._updatedAt, this.startAtTimestamp) >= 0,
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
if (this.startAfterValues) {
|
|
75
|
+
const [timestamp, documentId] = this.startAfterValues;
|
|
76
|
+
documents = documents.filter((document) => {
|
|
77
|
+
const timestampOrder = compareTimestamp(document.data()._updatedAt, timestamp);
|
|
78
|
+
return timestampOrder > 0 ||
|
|
79
|
+
(timestampOrder === 0 && document.id.localeCompare(documentId) > 0);
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
} else {
|
|
83
|
+
documents.sort((left, right) => left.id.localeCompare(right.id));
|
|
84
|
+
if (this.startAfterValues) {
|
|
85
|
+
const [document] = this.startAfterValues;
|
|
86
|
+
documents = documents.filter((candidate) => candidate.id > document.id);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
const page = documents.slice(0, this.pageSize);
|
|
90
|
+
return { docs: page, empty: page.length === 0, size: page.length };
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
class FakeFirestore {
|
|
95
|
+
constructor(collections) {
|
|
96
|
+
this.collections = collections;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
collection(name) {
|
|
100
|
+
return new FakeQuery(this.collections[name]);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
describe('incremental Firestore entity synchronization', () => {
|
|
105
|
+
it('updates derived rows, detects soft deletes, and stores no raw document IDs or text', async () => {
|
|
106
|
+
const temporaryDirectory = fs.mkdtempSync(
|
|
107
|
+
path.join(os.tmpdir(), 'analytics-entity-sync-'),
|
|
108
|
+
);
|
|
109
|
+
const cachePath = path.join(temporaryDirectory, 'cache.sqlite');
|
|
110
|
+
const cacheDb = await openDatabase(
|
|
111
|
+
cachePath,
|
|
112
|
+
sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE,
|
|
113
|
+
);
|
|
114
|
+
const collections = {
|
|
115
|
+
records: [
|
|
116
|
+
{
|
|
117
|
+
id: 'user-sensitive-id',
|
|
118
|
+
data: {
|
|
119
|
+
_updatedAt: new Timestamp(1, 0),
|
|
120
|
+
rawText: 'raw secret one',
|
|
121
|
+
value: 1,
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
id: 'second-id',
|
|
126
|
+
data: {
|
|
127
|
+
_updatedAt: new Timestamp(2, 0),
|
|
128
|
+
rawText: 'raw secret two',
|
|
129
|
+
value: 2,
|
|
130
|
+
},
|
|
131
|
+
},
|
|
132
|
+
],
|
|
133
|
+
};
|
|
134
|
+
const firestore = new FakeFirestore(collections);
|
|
135
|
+
const definition = {
|
|
136
|
+
analysisVersion: 1,
|
|
137
|
+
sources: [{
|
|
138
|
+
id: 'records',
|
|
139
|
+
collection: 'records',
|
|
140
|
+
derive: (data) => ({ derived_value: data.value }),
|
|
141
|
+
}],
|
|
142
|
+
materialize: (rows) => rows.map((row) => row.derived),
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
try {
|
|
146
|
+
const initialRows = await syncEntityDataset({
|
|
147
|
+
firestore,
|
|
148
|
+
cacheDb,
|
|
149
|
+
datasetId: 'test-dataset',
|
|
150
|
+
definition,
|
|
151
|
+
idSalt: 'private-salt',
|
|
152
|
+
pageSize: 1,
|
|
153
|
+
});
|
|
154
|
+
assert.deepEqual(
|
|
155
|
+
initialRows.map((row) => row.derived_value).sort(),
|
|
156
|
+
[1, 2],
|
|
157
|
+
);
|
|
158
|
+
|
|
159
|
+
collections.records = [
|
|
160
|
+
{
|
|
161
|
+
id: 'user-sensitive-id',
|
|
162
|
+
data: {
|
|
163
|
+
_updatedAt: new Timestamp(3, 0),
|
|
164
|
+
_deleted: true,
|
|
165
|
+
rawText: 'raw secret one',
|
|
166
|
+
value: 1,
|
|
167
|
+
},
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
id: 'second-id',
|
|
171
|
+
data: {
|
|
172
|
+
_updatedAt: new Timestamp(4, 0),
|
|
173
|
+
rawText: 'raw secret two',
|
|
174
|
+
value: 5,
|
|
175
|
+
},
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
id: 'third-id',
|
|
179
|
+
data: {
|
|
180
|
+
_updatedAt: new Timestamp(5, 0),
|
|
181
|
+
rawText: 'raw secret three',
|
|
182
|
+
value: 6,
|
|
183
|
+
},
|
|
184
|
+
},
|
|
185
|
+
];
|
|
186
|
+
|
|
187
|
+
const updatedRows = await syncEntityDataset({
|
|
188
|
+
firestore,
|
|
189
|
+
cacheDb,
|
|
190
|
+
datasetId: 'test-dataset',
|
|
191
|
+
definition,
|
|
192
|
+
idSalt: 'private-salt',
|
|
193
|
+
pageSize: 1,
|
|
194
|
+
});
|
|
195
|
+
assert.deepEqual(
|
|
196
|
+
updatedRows.map((row) => row.derived_value).sort(),
|
|
197
|
+
[5, 6],
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
const cached = await all(
|
|
201
|
+
cacheDb,
|
|
202
|
+
'SELECT document_key, derived_json FROM entity_derived_cache',
|
|
203
|
+
);
|
|
204
|
+
const serializedCache = JSON.stringify(cached);
|
|
205
|
+
assert.doesNotMatch(serializedCache, /user-sensitive-id/);
|
|
206
|
+
assert.doesNotMatch(serializedCache, /raw secret/);
|
|
207
|
+
} finally {
|
|
208
|
+
await closeDatabase(cacheDb);
|
|
209
|
+
fs.rmSync(temporaryDirectory, { recursive: true, force: true });
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
});
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pseudonymization Identifier Salt Resolver
|
|
3
|
+
*
|
|
4
|
+
* Generates or resolves a secret salt key to pseudonymize identifiers.
|
|
5
|
+
*
|
|
6
|
+
* Implementation Details:
|
|
7
|
+
* - Uses `flag: 'wx'` (atomic write) to avoid race conditions if multiple
|
|
8
|
+
* processes initialize simultaneously.
|
|
9
|
+
* - Stores local salt in `.local/analytics/id-salt` with 0o600 permissions.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { randomBytes } from 'node:crypto';
|
|
13
|
+
import fs from 'node:fs';
|
|
14
|
+
import path from 'node:path';
|
|
15
|
+
|
|
16
|
+
const saltFilename = 'id-salt';
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
function readSalt(filename) {
|
|
20
|
+
const salt = fs.readFileSync(filename, 'utf8').trim();
|
|
21
|
+
if (!salt) throw new Error(`Analytics identifier salt is empty: ${filename}`);
|
|
22
|
+
return salt;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @param {string} localRoot
|
|
27
|
+
* @param {string} [explicitSalt]
|
|
28
|
+
* @returns {{ value: string, source: string, generated?: boolean }}
|
|
29
|
+
*/
|
|
30
|
+
export function resolveIdentifierSalt(localRoot, explicitSalt) {
|
|
31
|
+
|
|
32
|
+
if (explicitSalt?.trim()) {
|
|
33
|
+
return { value: explicitSalt.trim(), source: 'environment' };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
fs.mkdirSync(localRoot, { recursive: true });
|
|
37
|
+
const filename = path.join(localRoot, saltFilename);
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
if (fs.existsSync(filename)) {
|
|
41
|
+
return { value: readSalt(filename), source: filename };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
const generated = randomBytes(32).toString('hex');
|
|
46
|
+
try {
|
|
47
|
+
fs.writeFileSync(filename, `${generated}\n`, {
|
|
48
|
+
encoding: 'utf8',
|
|
49
|
+
flag: 'wx', // Atomic exclusive write mode
|
|
50
|
+
mode: 0o600, // Read/write by owner only
|
|
51
|
+
});
|
|
52
|
+
return { value: generated, source: filename, generated: true };
|
|
53
|
+
} catch (error) {
|
|
54
|
+
if (error.code !== 'EEXIST') throw error;
|
|
55
|
+
return { value: readSalt(filename), source: filename };
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import test from 'node:test';
|
|
6
|
+
import { resolveIdentifierSalt } from './identifier_salt.js';
|
|
7
|
+
|
|
8
|
+
test('generates and then reuses a private local identifier salt', () => {
|
|
9
|
+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'analytics-salt-'));
|
|
10
|
+
try {
|
|
11
|
+
const first = resolveIdentifierSalt(directory);
|
|
12
|
+
const second = resolveIdentifierSalt(directory);
|
|
13
|
+
|
|
14
|
+
assert.equal(first.generated, true);
|
|
15
|
+
assert.equal(second.generated, undefined);
|
|
16
|
+
assert.equal(second.value, first.value);
|
|
17
|
+
assert.equal(fs.statSync(path.join(directory, 'id-salt')).mode & 0o777, 0o600);
|
|
18
|
+
} finally {
|
|
19
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test('uses an explicit environment salt without writing a file', () => {
|
|
24
|
+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'analytics-salt-'));
|
|
25
|
+
try {
|
|
26
|
+
const result = resolveIdentifierSalt(directory, 'configured-salt');
|
|
27
|
+
|
|
28
|
+
assert.deepEqual(result, {
|
|
29
|
+
value: 'configured-salt',
|
|
30
|
+
source: 'environment',
|
|
31
|
+
});
|
|
32
|
+
assert.equal(fs.existsSync(path.join(directory, 'id-salt')), false);
|
|
33
|
+
} finally {
|
|
34
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
35
|
+
}
|
|
36
|
+
});
|