@tech4good/angular-schematics 21.5.0 → 21.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +266 -1
- package/package.json +34 -7
- package/src/component/files/__name@dasherize__/__name@dasherize__.component.spec.ts.template +4 -5
- package/src/component/files/__name@dasherize__/__name@dasherize__.component.ts.template +1 -1
- package/src/entity/files/__name@dasherize__/__name@dasherize__.mock.ts.template +16 -7
- package/src/entity/files/__name@dasherize__/__name@dasherize__.model.ts.template +13 -6
- package/src/entity/files/__name@dasherize__/__name@dasherize__.store.ts.template +18 -10
- package/src/entity/index.js +61 -49
- package/src/entity/index.js.map +1 -1
- package/src/entity/schema.json +4 -0
- package/src/guard/files/__name@dasherize__.guard.spec.ts.template +0 -1
- package/src/ng-add/files/.github/workflows/angular-github-ci.yml.template +3 -0
- package/src/ng-add/files/.gitignore.template +5 -0
- package/src/ng-add/files/analytics/datasets.js.template +206 -0
- package/src/ng-add/files/analytics/entity_sources.js.template +58 -0
- package/src/ng-add/files/analytics/entity_sources.test.js.template +57 -0
- package/src/ng-add/files/analytics/lib/api_query.js.template +87 -0
- package/src/ng-add/files/analytics/lib/api_query.test.js.template +84 -0
- package/src/ng-add/files/analytics/lib/bigquery_query_compatibility.test.js.template +53 -0
- package/src/ng-add/files/analytics/lib/dataset_schema.js.template +53 -0
- package/src/ng-add/files/analytics/lib/dataset_schema.test.js.template +34 -0
- package/src/ng-add/files/analytics/lib/entity_sync.js.template +397 -0
- package/src/ng-add/files/analytics/lib/entity_sync.test.js.template +212 -0
- package/src/ng-add/files/analytics/lib/identifier_salt.js.template +57 -0
- package/src/ng-add/files/analytics/lib/identifier_salt.test.js.template +36 -0
- package/src/ng-add/files/analytics/lib/snapshot_files.js.template +46 -0
- package/src/ng-add/files/analytics/lib/snapshot_files.test.js.template +67 -0
- package/src/ng-add/files/analytics/lib/sqlite.js.template +92 -0
- package/src/ng-add/files/analytics/lib/upload_entities.js.template +81 -0
- package/src/ng-add/files/analytics/lib/upload_entities.test.js.template +86 -0
- package/src/ng-add/files/analytics/proxy.conf.json.template +7 -0
- package/src/ng-add/files/analytics/scripts/refresh_snapshot.js.template +518 -0
- package/src/ng-add/files/analytics/scripts/start.js.template +88 -0
- package/src/ng-add/files/analytics/server/server.js.template +212 -0
- package/src/ng-add/files/angular.json.template +80 -1
- package/src/ng-add/files/bigquery/queries/active_time_by_route.sql.template +27 -8
- package/src/ng-add/files/bigquery/queries/active_user_windows.sql.template +67 -0
- package/src/ng-add/files/bigquery/queries/dashboard_route_visits.sql.template +47 -0
- package/src/ng-add/files/bigquery/queries/data_quality.sql.template +323 -0
- package/src/ng-add/files/bigquery/queries/onboarding_funnel.sql.template +512 -0
- package/src/ng-add/files/bigquery/queries/onboarding_stage_observations.sql.template +134 -0
- package/src/ng-add/files/bigquery/queries/weekly_retention_cohorts.sql.template +90 -0
- package/src/ng-add/files/bigquery/scripts/export_entities.js.template +120 -0
- package/src/ng-add/files/bigquery/scripts/init_views.js.template +57 -0
- package/src/ng-add/files/bigquery/scripts/run_query.js.template +141 -0
- package/src/ng-add/files/bigquery/views/events_flat.sql.template +27 -12
- package/src/ng-add/files/bigquery/views/logs_latest.sql.template +16 -3
- package/src/ng-add/files/docs-infra/analytics-guide.md.template +270 -25
- package/src/ng-add/files/docs-infra/analytics-workbench-guide.md.template +307 -0
- package/src/ng-add/files/docs-infra/authentication-architecture.md.template +300 -0
- package/src/ng-add/files/docs-infra/email-infrastructure.md.template +43 -0
- package/src/ng-add/files/docs-infra/firestore-index-checker.md.template +87 -0
- package/src/ng-add/files/docs-infra/infrastructure-guide.md.template +123 -38
- package/src/ng-add/files/docs-infra/load-system-architecture.md.template +15 -1
- package/src/ng-add/files/docs-infra/querying-strategy.md.template +14 -0
- package/src/ng-add/files/docs-infra/stream-system-architecture.md.template +21 -7
- package/src/ng-add/files/eslint.config.js.template +22 -3
- package/src/ng-add/files/firebase.json.template +6 -0
- package/src/ng-add/files/firestore.indexes.json.template +23 -0
- package/src/ng-add/files/firestore.rules.spec.ts.template +84 -15
- package/src/ng-add/files/firestore.rules.template +29 -17
- package/src/ng-add/files/functions/.gitignore.template +3 -0
- package/src/ng-add/files/functions/package.json.template +13 -0
- package/src/ng-add/files/functions/src/email.ts.template +23 -0
- package/src/ng-add/files/functions/src/index.ts.template +44 -0
- package/src/ng-add/files/functions/tsconfig.json.template +13 -0
- package/src/ng-add/files/jest.config.ts.template +3 -2
- package/src/ng-add/files/package.json.template +41 -16
- package/src/ng-add/files/projects/analytics/src/app/analytics-api.service.ts.template +130 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.html.template +77 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.scss.template +240 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.ts.template +96 -0
- package/src/ng-add/files/projects/analytics/src/app/app.routes.ts.template +11 -0
- package/src/ng-add/files/projects/analytics/src/app/dashboard-pages.spec.ts.template +151 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.html.template +39 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.ts.template +95 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.html.template +47 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.ts.template +120 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.html.template +40 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.ts.template +98 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.html.template +35 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.ts.template +84 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.html.template +42 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.ts.template +110 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/analytics-utils.ts.template +43 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/chart-theme.ts.template +42 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-layout.scss.template +239 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.html.template +76 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.scss.template +128 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.spec.ts.template +31 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.ts.template +153 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.html.template +16 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.scss.template +63 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.ts.template +52 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.html.template +5 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.scss.template +56 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.spec.ts.template +30 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.ts.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.html.template +9 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.scss.template +43 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.ts.template +62 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.html.template +9 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.scss.template +33 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.ts.template +82 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.html.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.scss.template +45 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.ts.template +55 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.html.template +18 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.scss.template +45 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.ts.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/vega-chart/vega-chart.component.ts.template +121 -0
- package/src/ng-add/files/projects/analytics/src/index.html.template +19 -0
- package/src/ng-add/files/projects/analytics/src/main.ts.template +16 -0
- package/src/ng-add/files/projects/analytics/src/styles.scss.template +28 -0
- package/src/ng-add/files/projects/analytics/tsconfig.app.json.template +13 -0
- package/src/ng-add/files/projects/analytics/tsconfig.spec.json.template +15 -0
- package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.html.template +3 -1
- package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.spec.ts.template +10 -13
- package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.ts.template +2 -0
- package/src/ng-add/files/src/app/admin/admin-panel/admin-settings/admin-settings.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.spec.ts.template +11 -12
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.ts.template +7 -7
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.spec.ts.template +8 -10
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.ts.template +3 -2
- package/src/ng-add/files/src/app/app.component.html.template +3 -3
- package/src/ng-add/files/src/app/app.component.scss.template +1 -1
- package/src/ng-add/files/src/app/app.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/app.component.ts.template +3 -2
- package/src/ng-add/files/src/app/app.config.ts.template +21 -15
- package/src/ng-add/files/src/app/app.routes.ts.template +6 -0
- package/src/ng-add/files/src/app/auth-routing.spec.ts.template +13 -9
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.spec.ts.template +46 -29
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.ts.template +27 -18
- package/src/ng-add/files/src/app/core/auth/auth.providers.spec.ts.template +112 -0
- package/src/ng-add/files/src/app/core/auth/auth.providers.ts.template +48 -0
- package/src/ng-add/files/src/app/core/auth/auth.service.ts.template +14 -23
- package/src/ng-add/files/src/app/core/auth/guards/require-admin.guard.spec.ts.template +27 -0
- package/src/ng-add/files/src/app/core/auth/guards/require-admin.guard.ts.template +3 -2
- package/src/ng-add/files/src/app/core/auth/guards/require-auth-and-consent.guard.ts.template +7 -2
- package/src/ng-add/files/src/app/core/firebase/data.providers.prod.ts.template +21 -0
- package/src/ng-add/files/src/app/core/firebase/data.providers.ts.template +13 -0
- package/src/ng-add/files/src/app/core/local-storage/session.service.spec.ts.template +43 -0
- package/src/ng-add/files/src/app/core/local-storage/session.service.ts.template +7 -0
- package/src/ng-add/files/src/app/core/store/app-data.spec.ts.template +76 -0
- package/src/ng-add/files/src/app/core/store/auth/auth.mock.ts.template +20 -82
- package/src/ng-add/files/src/app/core/store/auth/auth.store.spec.ts.template +217 -0
- package/src/ng-add/files/src/app/core/store/auth/auth.store.ts.template +36 -172
- package/src/ng-add/files/src/app/core/store/user/user.mock.ts.template +18 -15
- package/src/ng-add/files/src/app/core/store/user/user.model.ts.template +7 -8
- package/src/ng-add/files/src/app/core/store/user/user.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/store/user-context/user-context.mock.ts.template +20 -8
- package/src/ng-add/files/src/app/core/store/user-context/user-context.model.ts.template +5 -6
- package/src/ng-add/files/src/app/core/store/user-context/user-context.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/store/user-private/user-private.mock.ts.template +20 -7
- package/src/ng-add/files/src/app/core/store/user-private/user-private.model.ts.template +6 -6
- package/src/ng-add/files/src/app/core/store/user-private/user-private.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/testing/app-mocks.ts.template +42 -0
- package/src/ng-add/files/src/app/first-time/auth-action/auth-action.component.html.template +4 -0
- package/src/ng-add/files/src/app/first-time/auth-action/auth-action.component.scss.template +1 -0
- package/src/ng-add/files/src/app/first-time/auth-action/auth-action.component.ts.template +38 -0
- package/src/ng-add/files/src/app/first-time/consent/consent.component.html.template +2 -1
- package/src/ng-add/files/src/app/first-time/consent/consent.component.scss.template +7 -3
- package/src/ng-add/files/src/app/first-time/consent/consent.component.spec.ts.template +31 -11
- package/src/ng-add/files/src/app/first-time/consent/consent.component.ts.template +16 -1
- package/src/ng-add/files/src/app/first-time/consent/consent.guard.ts.template +5 -1
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.spec.ts.template +27 -16
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.ts.template +102 -49
- package/src/ng-add/files/src/app/first-time/early-access/early-access.guard.ts.template +5 -1
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.html.template +2 -1
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.scss.template +7 -3
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.spec.ts.template +20 -6
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.ts.template +2 -0
- package/src/ng-add/files/src/app/first-time/early-access/waitlist-view/waitlist-view.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.html.template +18 -0
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.scss.template +2 -0
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.spec.ts.template +56 -0
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.ts.template +91 -0
- package/src/ng-add/files/src/app/first-time/first-time-routing.spec.ts.template +11 -7
- package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.html.template +19 -7
- package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.scss.template +51 -0
- package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.ts.template +44 -5
- package/src/ng-add/files/src/app/first-time/landing/landing.component.html.template +2 -1
- package/src/ng-add/files/src/app/first-time/landing/landing.component.scss.template +12 -8
- package/src/ng-add/files/src/app/first-time/landing/landing.component.spec.ts.template +33 -10
- package/src/ng-add/files/src/app/first-time/landing/landing.component.ts.template +6 -19
- package/src/ng-add/files/src/app/first-time/login/login.component.html.template +58 -6
- package/src/ng-add/files/src/app/first-time/login/login.component.scss.template +151 -22
- package/src/ng-add/files/src/app/first-time/login/login.component.spec.ts.template +30 -32
- package/src/ng-add/files/src/app/first-time/login/login.component.ts.template +85 -31
- package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.spec.ts.template +4 -2
- package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.ts.template +1 -1
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.html.template +12 -0
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.scss.template +39 -0
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.spec.ts.template +68 -0
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.ts.template +20 -0
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.html.template +24 -0
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.scss.template +50 -0
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.spec.ts.template +65 -0
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.ts.template +59 -0
- package/src/ng-add/files/src/app/main/account-profile/guest-deletion-dialog.component.ts.template +51 -0
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.html.template +50 -0
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.scss.template +137 -0
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.spec.ts.template +57 -0
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.ts.template +59 -0
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.html.template +133 -0
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.scss.template +207 -0
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.spec.ts.template +74 -0
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.ts.template +130 -0
- package/src/ng-add/files/src/app/main/home/home.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/main/home/home.component.ts.template +1 -6
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.html.template +15 -0
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.scss.template +79 -0
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.spec.ts.template +59 -0
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.ts.template +44 -0
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.html.template +45 -15
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.scss.template +103 -19
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.spec.ts.template +73 -9
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.ts.template +46 -18
- package/src/ng-add/files/src/environments/environment.prod.ts.template +1 -0
- package/src/ng-add/files/src/environments/environment.test.ts.template +1 -0
- package/src/ng-add/files/src/environments/environment.ts.template +1 -1
- package/src/ng-add/files/tsconfig.spec.json.template +3 -1
- package/src/ng-add/index.js +7 -0
- package/src/ng-add/index.js.map +1 -1
- package/src/shared/entity-registry.d.ts +8 -0
- package/src/shared/entity-registry.js +28 -0
- package/src/shared/entity-registry.js.map +1 -0
- package/tsconfig.json +2 -2
- package/.agents/skills/publish-npm/SKILL.md +0 -37
- package/.agents/skills/publish-npm/eval.yaml +0 -32
- package/.agents/skills/publish-npm/fixtures/package-v21.json.template +0 -8
- package/.husky/commit-msg +0 -1
- package/.husky/pre-commit +0 -1
- package/.t4g/agent-memory.db +0 -0
- package/src/component/index_spec.d.ts +0 -1
- package/src/component/index_spec.js +0 -68
- package/src/component/index_spec.js.map +0 -1
- package/src/entity/index_spec.d.ts +0 -1
- package/src/entity/index_spec.js +0 -153
- package/src/entity/index_spec.js.map +0 -1
- package/src/guard/index_spec.d.ts +0 -1
- package/src/guard/index_spec.js +0 -64
- package/src/guard/index_spec.js.map +0 -1
- package/src/ng-add/files/bigquery/scripts/export_entities.py.template +0 -143
- package/src/ng-add/files/bigquery/scripts/run_query.py.template +0 -167
- package/src/ng-add/files/src/app/core/firebase/cached-listeners.service.ts.template +0 -494
- package/src/ng-add/files/src/app/core/firebase/database.service.ts.template +0 -56
- package/src/ng-add/files/src/app/core/firebase/firebase.mock.service.ts.template +0 -366
- package/src/ng-add/files/src/app/core/firebase/firebase.service.ts.template +0 -342
- package/src/ng-add/files/src/app/core/firebase/simulate-load.spec.ts.template +0 -104
- package/src/ng-add/files/src/app/core/firebase/simulate-stream.spec.ts.template +0 -127
- package/src/ng-add/files/src/app/core/firebase/simulation.mock.ts.template +0 -38
- package/src/ng-add/files/src/app/core/firebase/stream-multiplexer.ts.template +0 -445
- package/src/ng-add/files/src/app/core/logging/logging.service.spec.ts.template +0 -118
- package/src/ng-add/files/src/app/core/logging/logging.service.ts.template +0 -168
- package/src/ng-add/files/src/app/core/store/app.model.ts.template +0 -21
- package/src/ng-add/files/src/app/core/store/app.store.spec.ts.template +0 -920
- package/src/ng-add/files/src/app/core/store/app.store.ts.template +0 -1317
- package/src/ng-add/files/src/app/core/store/batch-write.firebase.service.ts.template +0 -58
- package/src/ng-add/files/src/app/core/store/batch-write.mock.service.ts.template +0 -105
- package/src/ng-add/files/src/app/core/store/batch-write.service.ts.template +0 -126
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.html.template +0 -9
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.scss.template +0 -46
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.spec.ts.template +0 -69
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.ts.template +0 -39
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.spec.ts.template +0 -122
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.ts.template +0 -88
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import assert from 'node:assert/strict';
|
|
5
|
+
import { describe, it } from 'node:test';
|
|
6
|
+
import { Timestamp } from 'firebase-admin/firestore';
|
|
7
|
+
import sqlite3 from 'sqlite3';
|
|
8
|
+
import { all, closeDatabase, openDatabase } from './sqlite.js';
|
|
9
|
+
import { syncEntityDataset } from './entity_sync.js';
|
|
10
|
+
|
|
11
|
+
function compareTimestamp(left, right) {
|
|
12
|
+
if (left.seconds !== right.seconds) return left.seconds - right.seconds;
|
|
13
|
+
return left.nanoseconds - right.nanoseconds;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
class FakeDocument {
|
|
17
|
+
constructor(id, value) {
|
|
18
|
+
this.id = id;
|
|
19
|
+
this.value = value;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
data() {
|
|
23
|
+
return this.value;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
class FakeQuery {
|
|
28
|
+
constructor(documents) {
|
|
29
|
+
this.documents = documents;
|
|
30
|
+
this.updatedAtOrder = false;
|
|
31
|
+
this.startAtTimestamp = null;
|
|
32
|
+
this.startAfterValues = null;
|
|
33
|
+
this.pageSize = Number.POSITIVE_INFINITY;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
orderBy(field) {
|
|
37
|
+
if (field === '_updatedAt') this.updatedAtOrder = true;
|
|
38
|
+
return this;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
startAt(timestamp) {
|
|
42
|
+
this.startAtTimestamp = timestamp;
|
|
43
|
+
return this;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
startAfter(...values) {
|
|
47
|
+
this.startAfterValues = values;
|
|
48
|
+
return this;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
limit(value) {
|
|
52
|
+
this.pageSize = value;
|
|
53
|
+
return this;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async get() {
|
|
57
|
+
let documents = this.documents.map(
|
|
58
|
+
({ id, data }) => new FakeDocument(id, data),
|
|
59
|
+
);
|
|
60
|
+
if (this.updatedAtOrder) {
|
|
61
|
+
documents.sort((left, right) => {
|
|
62
|
+
const timestampOrder = compareTimestamp(
|
|
63
|
+
left.data()._updatedAt,
|
|
64
|
+
right.data()._updatedAt,
|
|
65
|
+
);
|
|
66
|
+
return timestampOrder || left.id.localeCompare(right.id);
|
|
67
|
+
});
|
|
68
|
+
if (this.startAtTimestamp) {
|
|
69
|
+
documents = documents.filter(
|
|
70
|
+
(document) =>
|
|
71
|
+
compareTimestamp(document.data()._updatedAt, this.startAtTimestamp) >= 0,
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
if (this.startAfterValues) {
|
|
75
|
+
const [timestamp, documentId] = this.startAfterValues;
|
|
76
|
+
documents = documents.filter((document) => {
|
|
77
|
+
const timestampOrder = compareTimestamp(document.data()._updatedAt, timestamp);
|
|
78
|
+
return timestampOrder > 0 ||
|
|
79
|
+
(timestampOrder === 0 && document.id.localeCompare(documentId) > 0);
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
} else {
|
|
83
|
+
documents.sort((left, right) => left.id.localeCompare(right.id));
|
|
84
|
+
if (this.startAfterValues) {
|
|
85
|
+
const [document] = this.startAfterValues;
|
|
86
|
+
documents = documents.filter((candidate) => candidate.id > document.id);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
const page = documents.slice(0, this.pageSize);
|
|
90
|
+
return { docs: page, empty: page.length === 0, size: page.length };
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
class FakeFirestore {
|
|
95
|
+
constructor(collections) {
|
|
96
|
+
this.collections = collections;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
collection(name) {
|
|
100
|
+
return new FakeQuery(this.collections[name]);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
describe('incremental Firestore entity synchronization', () => {
|
|
105
|
+
it('updates derived rows, detects soft deletes, and stores no raw document IDs or text', async () => {
|
|
106
|
+
const temporaryDirectory = fs.mkdtempSync(
|
|
107
|
+
path.join(os.tmpdir(), 'analytics-entity-sync-'),
|
|
108
|
+
);
|
|
109
|
+
const cachePath = path.join(temporaryDirectory, 'cache.sqlite');
|
|
110
|
+
const cacheDb = await openDatabase(
|
|
111
|
+
cachePath,
|
|
112
|
+
sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE,
|
|
113
|
+
);
|
|
114
|
+
const collections = {
|
|
115
|
+
records: [
|
|
116
|
+
{
|
|
117
|
+
id: 'user-sensitive-id',
|
|
118
|
+
data: {
|
|
119
|
+
_updatedAt: new Timestamp(1, 0),
|
|
120
|
+
rawText: 'raw secret one',
|
|
121
|
+
value: 1,
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
id: 'second-id',
|
|
126
|
+
data: {
|
|
127
|
+
_updatedAt: new Timestamp(2, 0),
|
|
128
|
+
rawText: 'raw secret two',
|
|
129
|
+
value: 2,
|
|
130
|
+
},
|
|
131
|
+
},
|
|
132
|
+
],
|
|
133
|
+
};
|
|
134
|
+
const firestore = new FakeFirestore(collections);
|
|
135
|
+
const definition = {
|
|
136
|
+
analysisVersion: 1,
|
|
137
|
+
sources: [{
|
|
138
|
+
id: 'records',
|
|
139
|
+
collection: 'records',
|
|
140
|
+
derive: (data) => ({ derived_value: data.value }),
|
|
141
|
+
}],
|
|
142
|
+
materialize: (rows) => rows.map((row) => row.derived),
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
try {
|
|
146
|
+
const initialRows = await syncEntityDataset({
|
|
147
|
+
firestore,
|
|
148
|
+
cacheDb,
|
|
149
|
+
datasetId: 'test-dataset',
|
|
150
|
+
definition,
|
|
151
|
+
idSalt: 'private-salt',
|
|
152
|
+
pageSize: 1,
|
|
153
|
+
});
|
|
154
|
+
assert.deepEqual(
|
|
155
|
+
initialRows.map((row) => row.derived_value).sort(),
|
|
156
|
+
[1, 2],
|
|
157
|
+
);
|
|
158
|
+
|
|
159
|
+
collections.records = [
|
|
160
|
+
{
|
|
161
|
+
id: 'user-sensitive-id',
|
|
162
|
+
data: {
|
|
163
|
+
_updatedAt: new Timestamp(3, 0),
|
|
164
|
+
_deleted: true,
|
|
165
|
+
rawText: 'raw secret one',
|
|
166
|
+
value: 1,
|
|
167
|
+
},
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
id: 'second-id',
|
|
171
|
+
data: {
|
|
172
|
+
_updatedAt: new Timestamp(4, 0),
|
|
173
|
+
rawText: 'raw secret two',
|
|
174
|
+
value: 5,
|
|
175
|
+
},
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
id: 'third-id',
|
|
179
|
+
data: {
|
|
180
|
+
_updatedAt: new Timestamp(5, 0),
|
|
181
|
+
rawText: 'raw secret three',
|
|
182
|
+
value: 6,
|
|
183
|
+
},
|
|
184
|
+
},
|
|
185
|
+
];
|
|
186
|
+
|
|
187
|
+
const updatedRows = await syncEntityDataset({
|
|
188
|
+
firestore,
|
|
189
|
+
cacheDb,
|
|
190
|
+
datasetId: 'test-dataset',
|
|
191
|
+
definition,
|
|
192
|
+
idSalt: 'private-salt',
|
|
193
|
+
pageSize: 1,
|
|
194
|
+
});
|
|
195
|
+
assert.deepEqual(
|
|
196
|
+
updatedRows.map((row) => row.derived_value).sort(),
|
|
197
|
+
[5, 6],
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
const cached = await all(
|
|
201
|
+
cacheDb,
|
|
202
|
+
'SELECT document_key, derived_json FROM entity_derived_cache',
|
|
203
|
+
);
|
|
204
|
+
const serializedCache = JSON.stringify(cached);
|
|
205
|
+
assert.doesNotMatch(serializedCache, /user-sensitive-id/);
|
|
206
|
+
assert.doesNotMatch(serializedCache, /raw secret/);
|
|
207
|
+
} finally {
|
|
208
|
+
await closeDatabase(cacheDb);
|
|
209
|
+
fs.rmSync(temporaryDirectory, { recursive: true, force: true });
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
});
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pseudonymization Identifier Salt Resolver
|
|
3
|
+
*
|
|
4
|
+
* Generates or resolves a secret salt key to pseudonymize identifiers.
|
|
5
|
+
*
|
|
6
|
+
* Implementation Details:
|
|
7
|
+
* - Uses `flag: 'wx'` (atomic write) to avoid race conditions if multiple
|
|
8
|
+
* processes initialize simultaneously.
|
|
9
|
+
* - Stores local salt in `.local/analytics/id-salt` with 0o600 permissions.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { randomBytes } from 'node:crypto';
|
|
13
|
+
import fs from 'node:fs';
|
|
14
|
+
import path from 'node:path';
|
|
15
|
+
|
|
16
|
+
const saltFilename = 'id-salt';
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
function readSalt(filename) {
|
|
20
|
+
const salt = fs.readFileSync(filename, 'utf8').trim();
|
|
21
|
+
if (!salt) throw new Error(`Analytics identifier salt is empty: ${filename}`);
|
|
22
|
+
return salt;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @param {string} localRoot
|
|
27
|
+
* @param {string} [explicitSalt]
|
|
28
|
+
* @returns {{ value: string, source: string, generated?: boolean }}
|
|
29
|
+
*/
|
|
30
|
+
export function resolveIdentifierSalt(localRoot, explicitSalt) {
|
|
31
|
+
|
|
32
|
+
if (explicitSalt?.trim()) {
|
|
33
|
+
return { value: explicitSalt.trim(), source: 'environment' };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
fs.mkdirSync(localRoot, { recursive: true });
|
|
37
|
+
const filename = path.join(localRoot, saltFilename);
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
if (fs.existsSync(filename)) {
|
|
41
|
+
return { value: readSalt(filename), source: filename };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
const generated = randomBytes(32).toString('hex');
|
|
46
|
+
try {
|
|
47
|
+
fs.writeFileSync(filename, `${generated}\n`, {
|
|
48
|
+
encoding: 'utf8',
|
|
49
|
+
flag: 'wx', // Atomic exclusive write mode
|
|
50
|
+
mode: 0o600, // Read/write by owner only
|
|
51
|
+
});
|
|
52
|
+
return { value: generated, source: filename, generated: true };
|
|
53
|
+
} catch (error) {
|
|
54
|
+
if (error.code !== 'EEXIST') throw error;
|
|
55
|
+
return { value: readSalt(filename), source: filename };
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import test from 'node:test';
|
|
6
|
+
import { resolveIdentifierSalt } from './identifier_salt.js';
|
|
7
|
+
|
|
8
|
+
test('generates and then reuses a private local identifier salt', () => {
|
|
9
|
+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'analytics-salt-'));
|
|
10
|
+
try {
|
|
11
|
+
const first = resolveIdentifierSalt(directory);
|
|
12
|
+
const second = resolveIdentifierSalt(directory);
|
|
13
|
+
|
|
14
|
+
assert.equal(first.generated, true);
|
|
15
|
+
assert.equal(second.generated, undefined);
|
|
16
|
+
assert.equal(second.value, first.value);
|
|
17
|
+
assert.equal(fs.statSync(path.join(directory, 'id-salt')).mode & 0o777, 0o600);
|
|
18
|
+
} finally {
|
|
19
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test('uses an explicit environment salt without writing a file', () => {
|
|
24
|
+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'analytics-salt-'));
|
|
25
|
+
try {
|
|
26
|
+
const result = resolveIdentifierSalt(directory, 'configured-salt');
|
|
27
|
+
|
|
28
|
+
assert.deepEqual(result, {
|
|
29
|
+
value: 'configured-salt',
|
|
30
|
+
source: 'environment',
|
|
31
|
+
});
|
|
32
|
+
assert.equal(fs.existsSync(path.join(directory, 'id-salt')), false);
|
|
33
|
+
} finally {
|
|
34
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
35
|
+
}
|
|
36
|
+
});
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic snapshot publishing.
|
|
3
|
+
*
|
|
4
|
+
* Uses rename-based atomic swaps so readers of `current.json` never
|
|
5
|
+
* see a half-written file.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import fs from 'node:fs';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* @param {object} params
|
|
12
|
+
* @param {string} params.buildingPath
|
|
13
|
+
* @param {string} params.snapshotPath
|
|
14
|
+
* @param {string} params.pointerBuildingPath
|
|
15
|
+
* @param {string} params.pointerPath
|
|
16
|
+
* @param {object} params.pointer
|
|
17
|
+
*/
|
|
18
|
+
export function publishSnapshot({
|
|
19
|
+
buildingPath,
|
|
20
|
+
snapshotPath,
|
|
21
|
+
pointerBuildingPath,
|
|
22
|
+
pointerPath,
|
|
23
|
+
pointer,
|
|
24
|
+
}) {
|
|
25
|
+
fs.renameSync(buildingPath, snapshotPath);
|
|
26
|
+
|
|
27
|
+
fs.writeFileSync(
|
|
28
|
+
pointerBuildingPath,
|
|
29
|
+
`${JSON.stringify(pointer, null, 2)}\n`,
|
|
30
|
+
'utf8',
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
fs.renameSync(pointerBuildingPath, pointerPath);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Removes temporary building files after a failed refresh.
|
|
38
|
+
* @param {object} params
|
|
39
|
+
* @param {string} params.buildingPath
|
|
40
|
+
* @param {string} params.pointerBuildingPath
|
|
41
|
+
*/
|
|
42
|
+
export function cleanupFailedSnapshot({ buildingPath, pointerBuildingPath }) {
|
|
43
|
+
for (const incompletePath of [buildingPath, pointerBuildingPath]) {
|
|
44
|
+
if (fs.existsSync(incompletePath)) fs.unlinkSync(incompletePath);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { describe, it } from 'node:test';
|
|
6
|
+
import {
|
|
7
|
+
cleanupFailedSnapshot,
|
|
8
|
+
publishSnapshot,
|
|
9
|
+
} from './snapshot_files.js';
|
|
10
|
+
|
|
11
|
+
describe('immutable snapshot publication', () => {
|
|
12
|
+
it('publishes a completed snapshot and atomically updates the pointer', () => {
|
|
13
|
+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'analytics-snapshot-'));
|
|
14
|
+
const buildingPath = path.join(directory, 'new.building');
|
|
15
|
+
const snapshotPath = path.join(directory, 'new.sqlite');
|
|
16
|
+
const pointerBuildingPath = path.join(directory, 'current.json.building');
|
|
17
|
+
const pointerPath = path.join(directory, 'current.json');
|
|
18
|
+
fs.writeFileSync(buildingPath, 'complete database');
|
|
19
|
+
fs.writeFileSync(pointerPath, '{"snapshotId":"old"}\n');
|
|
20
|
+
|
|
21
|
+
try {
|
|
22
|
+
publishSnapshot({
|
|
23
|
+
buildingPath,
|
|
24
|
+
snapshotPath,
|
|
25
|
+
pointerBuildingPath,
|
|
26
|
+
pointerPath,
|
|
27
|
+
pointer: { snapshotId: 'new', path: 'new.sqlite' },
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
assert.equal(fs.readFileSync(snapshotPath, 'utf8'), 'complete database');
|
|
31
|
+
assert.deepEqual(JSON.parse(fs.readFileSync(pointerPath, 'utf8')), {
|
|
32
|
+
snapshotId: 'new',
|
|
33
|
+
path: 'new.sqlite',
|
|
34
|
+
});
|
|
35
|
+
assert.equal(fs.existsSync(buildingPath), false);
|
|
36
|
+
assert.equal(fs.existsSync(pointerBuildingPath), false);
|
|
37
|
+
} finally {
|
|
38
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('removes incomplete files while preserving the last good snapshot', () => {
|
|
43
|
+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'analytics-snapshot-'));
|
|
44
|
+
const buildingPath = path.join(directory, 'new.building');
|
|
45
|
+
const pointerBuildingPath = path.join(directory, 'current.json.building');
|
|
46
|
+
const pointerPath = path.join(directory, 'current.json');
|
|
47
|
+
const oldSnapshotPath = path.join(directory, 'old.sqlite');
|
|
48
|
+
fs.writeFileSync(buildingPath, 'partial database');
|
|
49
|
+
fs.writeFileSync(pointerBuildingPath, '{"snapshotId":"new"}\n');
|
|
50
|
+
fs.writeFileSync(pointerPath, '{"snapshotId":"old","path":"old.sqlite"}\n');
|
|
51
|
+
fs.writeFileSync(oldSnapshotPath, 'last good database');
|
|
52
|
+
|
|
53
|
+
try {
|
|
54
|
+
cleanupFailedSnapshot({ buildingPath, pointerBuildingPath });
|
|
55
|
+
|
|
56
|
+
assert.equal(fs.existsSync(buildingPath), false);
|
|
57
|
+
assert.equal(fs.existsSync(pointerBuildingPath), false);
|
|
58
|
+
assert.deepEqual(JSON.parse(fs.readFileSync(pointerPath, 'utf8')), {
|
|
59
|
+
snapshotId: 'old',
|
|
60
|
+
path: 'old.sqlite',
|
|
61
|
+
});
|
|
62
|
+
assert.equal(fs.readFileSync(oldSnapshotPath, 'utf8'), 'last good database');
|
|
63
|
+
} finally {
|
|
64
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
});
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promise wrappers around the callback-based sqlite3 driver.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import sqlite3 from 'sqlite3';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @param {string} filename
|
|
9
|
+
* @param {number} mode
|
|
10
|
+
* @returns {Promise<sqlite3.Database>}
|
|
11
|
+
*/
|
|
12
|
+
export function openDatabase(filename, mode) {
|
|
13
|
+
return new Promise((resolve, reject) => {
|
|
14
|
+
const db = new sqlite3.Database(filename, mode, (error) => {
|
|
15
|
+
if (error) reject(error);
|
|
16
|
+
else resolve(db);
|
|
17
|
+
});
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* @param {sqlite3.Database} db
|
|
23
|
+
* @param {string} sql
|
|
24
|
+
* @param {Array<any>} params
|
|
25
|
+
* @returns {Promise<{ changes: number, lastID: number }>}
|
|
26
|
+
*/
|
|
27
|
+
export function run(db, sql, params = []) {
|
|
28
|
+
return new Promise((resolve, reject) => {
|
|
29
|
+
db.run(sql, params, function onRun(error) {
|
|
30
|
+
if (error) reject(error);
|
|
31
|
+
else resolve({ changes: this.changes, lastID: this.lastID });
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @param {sqlite3.Database} db
|
|
38
|
+
* @param {string} sql
|
|
39
|
+
* @param {Array<any>} params
|
|
40
|
+
* @returns {Promise<Array<object>>}
|
|
41
|
+
*/
|
|
42
|
+
export function all(db, sql, params = []) {
|
|
43
|
+
return new Promise((resolve, reject) => {
|
|
44
|
+
db.all(sql, params, (error, rows) => {
|
|
45
|
+
if (error) reject(error);
|
|
46
|
+
else resolve(rows);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* @param {sqlite3.Database} db
|
|
53
|
+
* @param {string} sql
|
|
54
|
+
* @param {Array<any>} params
|
|
55
|
+
* @returns {Promise<object|undefined>}
|
|
56
|
+
*/
|
|
57
|
+
export function get(db, sql, params = []) {
|
|
58
|
+
return new Promise((resolve, reject) => {
|
|
59
|
+
db.get(sql, params, (error, row) => {
|
|
60
|
+
if (error) reject(error);
|
|
61
|
+
else resolve(row);
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @param {sqlite3.Database} db
|
|
68
|
+
* @param {string} sql
|
|
69
|
+
* @returns {Promise<void>}
|
|
70
|
+
*/
|
|
71
|
+
export function exec(db, sql) {
|
|
72
|
+
return new Promise((resolve, reject) => {
|
|
73
|
+
db.exec(sql, (error) => {
|
|
74
|
+
if (error) reject(error);
|
|
75
|
+
else resolve();
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* @param {sqlite3.Database|null} db
|
|
82
|
+
* @returns {Promise<void>}
|
|
83
|
+
*/
|
|
84
|
+
export function closeDatabase(db) {
|
|
85
|
+
if (!db) return Promise.resolve();
|
|
86
|
+
return new Promise((resolve, reject) => {
|
|
87
|
+
db.close((error) => {
|
|
88
|
+
if (error) reject(error);
|
|
89
|
+
else resolve();
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Maps a SQLite column type to a BigQuery column type.
|
|
7
|
+
* @param {string} type - The SQLite column type (e.g., 'TEXT', 'INTEGER', 'REAL').
|
|
8
|
+
* @returns {string} The BigQuery column type ('STRING', 'INT64', 'FLOAT64').
|
|
9
|
+
* @throws {Error} If the type is not supported.
|
|
10
|
+
*/
|
|
11
|
+
export function mapSqliteTypeToBigQuery(type) {
|
|
12
|
+
switch (type.toUpperCase()) {
|
|
13
|
+
case 'TEXT':
|
|
14
|
+
return 'STRING';
|
|
15
|
+
case 'INTEGER':
|
|
16
|
+
return 'INT64';
|
|
17
|
+
case 'REAL':
|
|
18
|
+
return 'FLOAT64';
|
|
19
|
+
default:
|
|
20
|
+
throw new Error(`Unsupported SQLite type: ${type}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Uploads entity rows to a BigQuery table, fully replacing the existing table data.
|
|
26
|
+
* The table will be created if it does not exist.
|
|
27
|
+
*
|
|
28
|
+
* @param {Object} options - Upload options.
|
|
29
|
+
* @param {import('@google-cloud/bigquery').BigQuery} options.bigquery - The BigQuery client instance.
|
|
30
|
+
* @param {string} options.datasetId - The name of the BigQuery dataset.
|
|
31
|
+
* @param {string} options.tableName - The name of the BigQuery table.
|
|
32
|
+
* @param {Array<{name: string, type: string}>} options.columns - The array of column definitions.
|
|
33
|
+
* @param {Array<Object>} options.rows - The array of row objects to upload.
|
|
34
|
+
* @returns {Promise<Object>} The completed BigQuery load job metadata.
|
|
35
|
+
*/
|
|
36
|
+
export async function uploadEntityTable({ bigquery, datasetId, tableName, columns, rows }) {
|
|
37
|
+
// Convert columns to BigQuery schema
|
|
38
|
+
const schema = columns.map(col => ({
|
|
39
|
+
name: col.name,
|
|
40
|
+
type: mapSqliteTypeToBigQuery(col.type),
|
|
41
|
+
}));
|
|
42
|
+
|
|
43
|
+
// Convert rows to NDJSON format
|
|
44
|
+
// We use join('\n') and handle empty arrays safely (it will return empty string)
|
|
45
|
+
const ndjson = rows.map(row => JSON.stringify(row)).join('\n');
|
|
46
|
+
|
|
47
|
+
// Write NDJSON to a temporary file
|
|
48
|
+
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'bq-upload-'));
|
|
49
|
+
const tempFilePath = path.join(tempDir, `${tableName}.json`);
|
|
50
|
+
|
|
51
|
+
try {
|
|
52
|
+
await fs.writeFile(tempFilePath, ndjson);
|
|
53
|
+
|
|
54
|
+
// Load into BigQuery
|
|
55
|
+
const dataset = bigquery.dataset(datasetId);
|
|
56
|
+
const table = dataset.table(tableName);
|
|
57
|
+
|
|
58
|
+
// table.load() polls the underlying job and resolves with its final
|
|
59
|
+
// metadata. It does not return the Job instance created by
|
|
60
|
+
// table.createLoadJob().
|
|
61
|
+
const [metadata] = await table.load(tempFilePath, {
|
|
62
|
+
sourceFormat: 'NEWLINE_DELIMITED_JSON',
|
|
63
|
+
writeDisposition: 'WRITE_TRUNCATE',
|
|
64
|
+
autodetect: false,
|
|
65
|
+
schema: { fields: schema },
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
// Check for load errors after the job completes
|
|
69
|
+
const status = metadata.status;
|
|
70
|
+
if (status.errors && status.errors.length > 0) {
|
|
71
|
+
throw new Error(
|
|
72
|
+
`BigQuery load job for ${tableName} failed: ${JSON.stringify(status.errors)}`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
return metadata;
|
|
77
|
+
} finally {
|
|
78
|
+
// Clean up temp file and directory
|
|
79
|
+
await fs.rm(tempDir, { recursive: true, force: true });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import assert from 'node:assert';
|
|
2
|
+
import test from 'node:test';
|
|
3
|
+
import { mapSqliteTypeToBigQuery, uploadEntityTable } from './upload_entities.js';
|
|
4
|
+
import fs from 'node:fs/promises';
|
|
5
|
+
|
|
6
|
+
test('mapSqliteTypeToBigQuery correctly maps types', () => {
|
|
7
|
+
assert.strictEqual(mapSqliteTypeToBigQuery('TEXT'), 'STRING');
|
|
8
|
+
assert.strictEqual(mapSqliteTypeToBigQuery('INTEGER'), 'INT64');
|
|
9
|
+
assert.strictEqual(mapSqliteTypeToBigQuery('REAL'), 'FLOAT64');
|
|
10
|
+
assert.strictEqual(mapSqliteTypeToBigQuery('text'), 'STRING');
|
|
11
|
+
|
|
12
|
+
assert.throws(() => mapSqliteTypeToBigQuery('BLOB'), /Unsupported SQLite type: BLOB/);
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test('uploadEntityTable prepares data and calls bigquery correctly', async () => {
|
|
16
|
+
const columns = [
|
|
17
|
+
{ name: 'id', type: 'INTEGER' },
|
|
18
|
+
{ name: 'name', type: 'TEXT' },
|
|
19
|
+
{ name: 'score', type: 'REAL' }
|
|
20
|
+
];
|
|
21
|
+
|
|
22
|
+
const rows = [
|
|
23
|
+
{ id: 1, name: 'Alice', score: 95.5 },
|
|
24
|
+
{ id: 2, name: 'Bob', score: 82.0 }
|
|
25
|
+
];
|
|
26
|
+
|
|
27
|
+
let loadedFilePath = null;
|
|
28
|
+
let loadedMetadata = null;
|
|
29
|
+
|
|
30
|
+
const mockTable = {
|
|
31
|
+
load: async (filePath, metadata) => {
|
|
32
|
+
loadedFilePath = filePath;
|
|
33
|
+
loadedMetadata = metadata;
|
|
34
|
+
|
|
35
|
+
// Verify file content during load
|
|
36
|
+
const content = await fs.readFile(filePath, 'utf-8');
|
|
37
|
+
assert.strictEqual(
|
|
38
|
+
content,
|
|
39
|
+
'{"id":1,"name":"Alice","score":95.5}\n{"id":2,"name":"Bob","score":82}'
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
return [{ status: { state: 'DONE', errors: [] } }];
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
const mockDataset = {
|
|
47
|
+
table: (tableName) => {
|
|
48
|
+
assert.strictEqual(tableName, 'test_table');
|
|
49
|
+
return mockTable;
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
const mockBigquery = {
|
|
54
|
+
dataset: (datasetId) => {
|
|
55
|
+
assert.strictEqual(datasetId, 'test_dataset');
|
|
56
|
+
return mockDataset;
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const metadata = await uploadEntityTable({
|
|
61
|
+
bigquery: mockBigquery,
|
|
62
|
+
datasetId: 'test_dataset',
|
|
63
|
+
tableName: 'test_table',
|
|
64
|
+
columns,
|
|
65
|
+
rows
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
assert.deepStrictEqual(metadata, { status: { state: 'DONE', errors: [] } });
|
|
69
|
+
|
|
70
|
+
assert.deepStrictEqual(loadedMetadata, {
|
|
71
|
+
sourceFormat: 'NEWLINE_DELIMITED_JSON',
|
|
72
|
+
writeDisposition: 'WRITE_TRUNCATE',
|
|
73
|
+
autodetect: false,
|
|
74
|
+
schema: {
|
|
75
|
+
fields: [
|
|
76
|
+
{ name: 'id', type: 'INT64' },
|
|
77
|
+
{ name: 'name', type: 'STRING' },
|
|
78
|
+
{ name: 'score', type: 'FLOAT64' }
|
|
79
|
+
]
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
// Verify temp file is cleaned up
|
|
84
|
+
const fileExists = await fs.access(loadedFilePath).then(() => true).catch(() => false);
|
|
85
|
+
assert.strictEqual(fileExists, false);
|
|
86
|
+
});
|