@tech4good/angular-schematics 21.6.0 → 21.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +266 -1
- package/package.json +34 -7
- package/src/component/files/__name@dasherize__/__name@dasherize__.component.spec.ts.template +4 -5
- package/src/component/files/__name@dasherize__/__name@dasherize__.component.ts.template +1 -1
- package/src/entity/files/__name@dasherize__/__name@dasherize__.mock.ts.template +14 -6
- package/src/entity/files/__name@dasherize__/__name@dasherize__.model.ts.template +12 -6
- package/src/entity/files/__name@dasherize__/__name@dasherize__.store.ts.template +18 -10
- package/src/entity/index.js +4 -49
- package/src/entity/index.js.map +1 -1
- package/src/guard/files/__name@dasherize__.guard.spec.ts.template +0 -1
- package/src/ng-add/files/.github/workflows/angular-github-ci.yml.template +3 -0
- package/src/ng-add/files/.gitignore.template +3 -0
- package/src/ng-add/files/analytics/datasets.js.template +206 -0
- package/src/ng-add/files/analytics/entity_sources.js.template +58 -0
- package/src/ng-add/files/analytics/entity_sources.test.js.template +57 -0
- package/src/ng-add/files/analytics/lib/api_query.js.template +87 -0
- package/src/ng-add/files/analytics/lib/api_query.test.js.template +84 -0
- package/src/ng-add/files/analytics/lib/bigquery_query_compatibility.test.js.template +53 -0
- package/src/ng-add/files/analytics/lib/dataset_schema.js.template +53 -0
- package/src/ng-add/files/analytics/lib/dataset_schema.test.js.template +34 -0
- package/src/ng-add/files/analytics/lib/entity_sync.js.template +397 -0
- package/src/ng-add/files/analytics/lib/entity_sync.test.js.template +212 -0
- package/src/ng-add/files/analytics/lib/identifier_salt.js.template +57 -0
- package/src/ng-add/files/analytics/lib/identifier_salt.test.js.template +36 -0
- package/src/ng-add/files/analytics/lib/snapshot_files.js.template +46 -0
- package/src/ng-add/files/analytics/lib/snapshot_files.test.js.template +67 -0
- package/src/ng-add/files/analytics/lib/sqlite.js.template +92 -0
- package/src/ng-add/files/analytics/lib/upload_entities.js.template +81 -0
- package/src/ng-add/files/analytics/lib/upload_entities.test.js.template +86 -0
- package/src/ng-add/files/analytics/proxy.conf.json.template +7 -0
- package/src/ng-add/files/analytics/scripts/refresh_snapshot.js.template +518 -0
- package/src/ng-add/files/analytics/scripts/start.js.template +88 -0
- package/src/ng-add/files/analytics/server/server.js.template +212 -0
- package/src/ng-add/files/angular.json.template +80 -1
- package/src/ng-add/files/bigquery/queries/active_time_by_route.sql.template +27 -8
- package/src/ng-add/files/bigquery/queries/active_user_windows.sql.template +67 -0
- package/src/ng-add/files/bigquery/queries/dashboard_route_visits.sql.template +47 -0
- package/src/ng-add/files/bigquery/queries/data_quality.sql.template +323 -0
- package/src/ng-add/files/bigquery/queries/onboarding_funnel.sql.template +512 -0
- package/src/ng-add/files/bigquery/queries/onboarding_stage_observations.sql.template +134 -0
- package/src/ng-add/files/bigquery/queries/weekly_retention_cohorts.sql.template +90 -0
- package/src/ng-add/files/bigquery/scripts/export_entities.js.template +120 -0
- package/src/ng-add/files/bigquery/scripts/init_views.js.template +57 -0
- package/src/ng-add/files/bigquery/scripts/run_query.js.template +141 -0
- package/src/ng-add/files/bigquery/views/events_flat.sql.template +27 -12
- package/src/ng-add/files/bigquery/views/logs_latest.sql.template +16 -3
- package/src/ng-add/files/docs-infra/analytics-guide.md.template +270 -25
- package/src/ng-add/files/docs-infra/analytics-workbench-guide.md.template +307 -0
- package/src/ng-add/files/docs-infra/authentication-architecture.md.template +54 -24
- package/src/ng-add/files/docs-infra/email-infrastructure.md.template +43 -0
- package/src/ng-add/files/docs-infra/firestore-index-checker.md.template +87 -0
- package/src/ng-add/files/docs-infra/infrastructure-guide.md.template +108 -29
- package/src/ng-add/files/docs-infra/load-system-architecture.md.template +15 -1
- package/src/ng-add/files/docs-infra/querying-strategy.md.template +14 -0
- package/src/ng-add/files/docs-infra/stream-system-architecture.md.template +21 -7
- package/src/ng-add/files/eslint.config.js.template +22 -3
- package/src/ng-add/files/functions/package.json.template +1 -0
- package/src/ng-add/files/functions/src/email.ts.template +23 -0
- package/src/ng-add/files/jest.config.ts.template +3 -2
- package/src/ng-add/files/package.json.template +39 -16
- package/src/ng-add/files/projects/analytics/src/app/analytics-api.service.ts.template +130 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.html.template +77 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.scss.template +240 -0
- package/src/ng-add/files/projects/analytics/src/app/app.component.ts.template +96 -0
- package/src/ng-add/files/projects/analytics/src/app/app.routes.ts.template +11 -0
- package/src/ng-add/files/projects/analytics/src/app/dashboard-pages.spec.ts.template +151 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.html.template +39 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/data-quality/data-quality-page/data-quality-page.component.ts.template +95 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.html.template +47 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/engagement/engagement-page/engagement-page.component.ts.template +120 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.html.template +40 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/onboarding/onboarding-page/onboarding-page.component.ts.template +98 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.html.template +35 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/overview/overview-page/overview-page.component.ts.template +84 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.html.template +42 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.scss.template +1 -0
- package/src/ng-add/files/projects/analytics/src/app/routes/routes-page/routes-page.component.ts.template +110 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/analytics-utils.ts.template +43 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/chart-theme.ts.template +42 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-layout.scss.template +239 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.html.template +76 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.scss.template +128 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.spec.ts.template +31 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/dashboard-table/dashboard-table.component.ts.template +153 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.html.template +16 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.scss.template +63 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/funnel-chart/funnel-chart.component.ts.template +52 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.html.template +5 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.scss.template +56 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.spec.ts.template +30 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/metric-card/metric-card.component.ts.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.html.template +9 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.scss.template +43 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/quality-chart/quality-chart.component.ts.template +62 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.html.template +9 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.scss.template +33 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-distribution-chart/route-distribution-chart.component.ts.template +82 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.html.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.scss.template +45 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/route-traffic-chart/route-traffic-chart.component.ts.template +55 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.html.template +18 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.scss.template +45 -0
- package/src/ng-add/files/projects/analytics/src/app/shared/status-message/status-message.component.ts.template +15 -0
- package/src/ng-add/files/projects/analytics/src/app/vega-chart/vega-chart.component.ts.template +121 -0
- package/src/ng-add/files/projects/analytics/src/index.html.template +19 -0
- package/src/ng-add/files/projects/analytics/src/main.ts.template +16 -0
- package/src/ng-add/files/projects/analytics/src/styles.scss.template +28 -0
- package/src/ng-add/files/projects/analytics/tsconfig.app.json.template +13 -0
- package/src/ng-add/files/projects/analytics/tsconfig.spec.json.template +15 -0
- package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/admin/admin-panel/admin-settings/admin-settings.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.spec.ts.template +11 -12
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-list.component.ts.template +7 -7
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.spec.ts.template +8 -10
- package/src/ng-add/files/src/app/admin/admin-panel/admin-user-list/admin-user-table/admin-user-table.component.ts.template +3 -2
- package/src/ng-add/files/src/app/app.component.html.template +1 -1
- package/src/ng-add/files/src/app/app.component.scss.template +1 -1
- package/src/ng-add/files/src/app/app.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/app.component.ts.template +1 -2
- package/src/ng-add/files/src/app/app.config.ts.template +21 -15
- package/src/ng-add/files/src/app/auth-routing.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.spec.ts.template +46 -29
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.ts.template +27 -18
- package/src/ng-add/files/src/app/core/auth/auth.providers.spec.ts.template +112 -0
- package/src/ng-add/files/src/app/core/auth/auth.providers.ts.template +48 -0
- package/src/ng-add/files/src/app/core/auth/auth.service.ts.template +9 -23
- package/src/ng-add/files/src/app/core/auth/guards/require-admin.guard.spec.ts.template +3 -14
- package/src/ng-add/files/src/app/core/firebase/data.providers.prod.ts.template +21 -0
- package/src/ng-add/files/src/app/core/firebase/data.providers.ts.template +13 -0
- package/src/ng-add/files/src/app/core/local-storage/session.service.spec.ts.template +43 -0
- package/src/ng-add/files/src/app/core/local-storage/session.service.ts.template +7 -0
- package/src/ng-add/files/src/app/core/store/app-data.spec.ts.template +76 -0
- package/src/ng-add/files/src/app/core/store/auth/auth.mock.ts.template +19 -67
- package/src/ng-add/files/src/app/core/store/auth/auth.store.spec.ts.template +102 -12
- package/src/ng-add/files/src/app/core/store/auth/auth.store.ts.template +32 -237
- package/src/ng-add/files/src/app/core/store/user/user.mock.ts.template +17 -12
- package/src/ng-add/files/src/app/core/store/user/user.model.ts.template +6 -7
- package/src/ng-add/files/src/app/core/store/user/user.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/store/user-context/user-context.mock.ts.template +20 -8
- package/src/ng-add/files/src/app/core/store/user-context/user-context.model.ts.template +5 -6
- package/src/ng-add/files/src/app/core/store/user-context/user-context.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/store/user-private/user-private.mock.ts.template +20 -7
- package/src/ng-add/files/src/app/core/store/user-private/user-private.model.ts.template +5 -6
- package/src/ng-add/files/src/app/core/store/user-private/user-private.store.ts.template +14 -10
- package/src/ng-add/files/src/app/core/testing/app-mocks.ts.template +42 -0
- package/src/ng-add/files/src/app/first-time/consent/consent.component.html.template +2 -1
- package/src/ng-add/files/src/app/first-time/consent/consent.component.scss.template +7 -3
- package/src/ng-add/files/src/app/first-time/consent/consent.component.spec.ts.template +31 -11
- package/src/ng-add/files/src/app/first-time/consent/consent.component.ts.template +16 -1
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.spec.ts.template +15 -15
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.ts.template +11 -3
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.html.template +2 -1
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.scss.template +7 -3
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.spec.ts.template +20 -6
- package/src/ng-add/files/src/app/first-time/early-access/survey-form/survey-form.component.ts.template +2 -0
- package/src/ng-add/files/src/app/first-time/early-access/waitlist-view/waitlist-view.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.spec.ts.template +17 -1
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.ts.template +15 -0
- package/src/ng-add/files/src/app/first-time/first-time-routing.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.ts.template +15 -0
- package/src/ng-add/files/src/app/first-time/landing/landing.component.spec.ts.template +2 -0
- package/src/ng-add/files/src/app/first-time/login/login.component.spec.ts.template +21 -3
- package/src/ng-add/files/src/app/first-time/login/login.component.ts.template +26 -0
- package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.spec.ts.template +4 -2
- package/src/ng-add/files/src/app/first-time/onboarding/onboarding.component.ts.template +1 -1
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.spec.ts.template +4 -4
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.spec.ts.template +2 -2
- package/src/ng-add/files/src/app/main/home/home.component.spec.ts.template +4 -5
- package/src/ng-add/files/src/app/main/home/home.component.ts.template +1 -1
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.spec.ts.template +4 -4
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.spec.ts.template +2 -2
- package/src/ng-add/files/tsconfig.spec.json.template +3 -1
- package/src/ng-add/index.js +7 -0
- package/src/ng-add/index.js.map +1 -1
- package/src/shared/entity-registry.d.ts +8 -0
- package/src/shared/entity-registry.js +28 -0
- package/src/shared/entity-registry.js.map +1 -0
- package/.agents/skills/publish-npm/SKILL.md +0 -37
- package/.agents/skills/publish-npm/eval.yaml +0 -32
- package/.agents/skills/publish-npm/fixtures/package-v21.json.template +0 -8
- package/.husky/commit-msg +0 -1
- package/.husky/pre-commit +0 -1
- package/.t4g/agent-memory.db +0 -0
- package/src/component/index_spec.d.ts +0 -1
- package/src/component/index_spec.js +0 -68
- package/src/component/index_spec.js.map +0 -1
- package/src/entity/index_spec.d.ts +0 -1
- package/src/entity/index_spec.js +0 -153
- package/src/entity/index_spec.js.map +0 -1
- package/src/guard/index_spec.d.ts +0 -1
- package/src/guard/index_spec.js +0 -64
- package/src/guard/index_spec.js.map +0 -1
- package/src/ng-add/files/bigquery/scripts/export_entities.py.template +0 -143
- package/src/ng-add/files/bigquery/scripts/run_query.py.template +0 -167
- package/src/ng-add/files/src/app/core/firebase/cached-listeners.service.ts.template +0 -500
- package/src/ng-add/files/src/app/core/firebase/database.service.ts.template +0 -73
- package/src/ng-add/files/src/app/core/firebase/email-link-context.service.spec.ts.template +0 -59
- package/src/ng-add/files/src/app/core/firebase/email-link-context.service.ts.template +0 -147
- package/src/ng-add/files/src/app/core/firebase/firebase.mock.service.ts.template +0 -453
- package/src/ng-add/files/src/app/core/firebase/firebase.service.spec.ts.template +0 -116
- package/src/ng-add/files/src/app/core/firebase/firebase.service.ts.template +0 -639
- package/src/ng-add/files/src/app/core/firebase/simulate-load.spec.ts.template +0 -104
- package/src/ng-add/files/src/app/core/firebase/simulate-stream.spec.ts.template +0 -127
- package/src/ng-add/files/src/app/core/firebase/simulation.mock.ts.template +0 -38
- package/src/ng-add/files/src/app/core/firebase/stream-multiplexer.ts.template +0 -445
- package/src/ng-add/files/src/app/core/logging/logging.service.spec.ts.template +0 -118
- package/src/ng-add/files/src/app/core/logging/logging.service.ts.template +0 -168
- package/src/ng-add/files/src/app/core/store/app.model.ts.template +0 -21
- package/src/ng-add/files/src/app/core/store/app.store.spec.ts.template +0 -920
- package/src/ng-add/files/src/app/core/store/app.store.ts.template +0 -1320
- package/src/ng-add/files/src/app/core/store/batch-write.firebase.service.ts.template +0 -58
- package/src/ng-add/files/src/app/core/store/batch-write.mock.service.ts.template +0 -105
- package/src/ng-add/files/src/app/core/store/batch-write.service.ts.template +0 -126
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.html.template +0 -9
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.scss.template +0 -46
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.spec.ts.template +0 -69
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.component.ts.template +0 -39
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.spec.ts.template +0 -122
- package/src/ng-add/files/src/app/shared/redeployment-warning/redeployment-warning.service.ts.template +0 -88
- package/src/ng-add/index_spec.d.ts +0 -1
- package/src/ng-add/index_spec.js +0 -108
- package/src/ng-add/index_spec.js.map +0 -1
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic snapshot publishing.
|
|
3
|
+
*
|
|
4
|
+
* Uses rename-based atomic swaps so readers of `current.json` never
|
|
5
|
+
* see a half-written file.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import fs from 'node:fs';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* @param {object} params
|
|
12
|
+
* @param {string} params.buildingPath
|
|
13
|
+
* @param {string} params.snapshotPath
|
|
14
|
+
* @param {string} params.pointerBuildingPath
|
|
15
|
+
* @param {string} params.pointerPath
|
|
16
|
+
* @param {object} params.pointer
|
|
17
|
+
*/
|
|
18
|
+
export function publishSnapshot({
|
|
19
|
+
buildingPath,
|
|
20
|
+
snapshotPath,
|
|
21
|
+
pointerBuildingPath,
|
|
22
|
+
pointerPath,
|
|
23
|
+
pointer,
|
|
24
|
+
}) {
|
|
25
|
+
fs.renameSync(buildingPath, snapshotPath);
|
|
26
|
+
|
|
27
|
+
fs.writeFileSync(
|
|
28
|
+
pointerBuildingPath,
|
|
29
|
+
`${JSON.stringify(pointer, null, 2)}\n`,
|
|
30
|
+
'utf8',
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
fs.renameSync(pointerBuildingPath, pointerPath);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Removes temporary building files after a failed refresh.
|
|
38
|
+
* @param {object} params
|
|
39
|
+
* @param {string} params.buildingPath
|
|
40
|
+
* @param {string} params.pointerBuildingPath
|
|
41
|
+
*/
|
|
42
|
+
export function cleanupFailedSnapshot({ buildingPath, pointerBuildingPath }) {
|
|
43
|
+
for (const incompletePath of [buildingPath, pointerBuildingPath]) {
|
|
44
|
+
if (fs.existsSync(incompletePath)) fs.unlinkSync(incompletePath);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { describe, it } from 'node:test';
|
|
6
|
+
import {
|
|
7
|
+
cleanupFailedSnapshot,
|
|
8
|
+
publishSnapshot,
|
|
9
|
+
} from './snapshot_files.js';
|
|
10
|
+
|
|
11
|
+
describe('immutable snapshot publication', () => {
|
|
12
|
+
it('publishes a completed snapshot and atomically updates the pointer', () => {
|
|
13
|
+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'analytics-snapshot-'));
|
|
14
|
+
const buildingPath = path.join(directory, 'new.building');
|
|
15
|
+
const snapshotPath = path.join(directory, 'new.sqlite');
|
|
16
|
+
const pointerBuildingPath = path.join(directory, 'current.json.building');
|
|
17
|
+
const pointerPath = path.join(directory, 'current.json');
|
|
18
|
+
fs.writeFileSync(buildingPath, 'complete database');
|
|
19
|
+
fs.writeFileSync(pointerPath, '{"snapshotId":"old"}\n');
|
|
20
|
+
|
|
21
|
+
try {
|
|
22
|
+
publishSnapshot({
|
|
23
|
+
buildingPath,
|
|
24
|
+
snapshotPath,
|
|
25
|
+
pointerBuildingPath,
|
|
26
|
+
pointerPath,
|
|
27
|
+
pointer: { snapshotId: 'new', path: 'new.sqlite' },
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
assert.equal(fs.readFileSync(snapshotPath, 'utf8'), 'complete database');
|
|
31
|
+
assert.deepEqual(JSON.parse(fs.readFileSync(pointerPath, 'utf8')), {
|
|
32
|
+
snapshotId: 'new',
|
|
33
|
+
path: 'new.sqlite',
|
|
34
|
+
});
|
|
35
|
+
assert.equal(fs.existsSync(buildingPath), false);
|
|
36
|
+
assert.equal(fs.existsSync(pointerBuildingPath), false);
|
|
37
|
+
} finally {
|
|
38
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('removes incomplete files while preserving the last good snapshot', () => {
|
|
43
|
+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'analytics-snapshot-'));
|
|
44
|
+
const buildingPath = path.join(directory, 'new.building');
|
|
45
|
+
const pointerBuildingPath = path.join(directory, 'current.json.building');
|
|
46
|
+
const pointerPath = path.join(directory, 'current.json');
|
|
47
|
+
const oldSnapshotPath = path.join(directory, 'old.sqlite');
|
|
48
|
+
fs.writeFileSync(buildingPath, 'partial database');
|
|
49
|
+
fs.writeFileSync(pointerBuildingPath, '{"snapshotId":"new"}\n');
|
|
50
|
+
fs.writeFileSync(pointerPath, '{"snapshotId":"old","path":"old.sqlite"}\n');
|
|
51
|
+
fs.writeFileSync(oldSnapshotPath, 'last good database');
|
|
52
|
+
|
|
53
|
+
try {
|
|
54
|
+
cleanupFailedSnapshot({ buildingPath, pointerBuildingPath });
|
|
55
|
+
|
|
56
|
+
assert.equal(fs.existsSync(buildingPath), false);
|
|
57
|
+
assert.equal(fs.existsSync(pointerBuildingPath), false);
|
|
58
|
+
assert.deepEqual(JSON.parse(fs.readFileSync(pointerPath, 'utf8')), {
|
|
59
|
+
snapshotId: 'old',
|
|
60
|
+
path: 'old.sqlite',
|
|
61
|
+
});
|
|
62
|
+
assert.equal(fs.readFileSync(oldSnapshotPath, 'utf8'), 'last good database');
|
|
63
|
+
} finally {
|
|
64
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
});
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promise wrappers around the callback-based sqlite3 driver.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import sqlite3 from 'sqlite3';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @param {string} filename
|
|
9
|
+
* @param {number} mode
|
|
10
|
+
* @returns {Promise<sqlite3.Database>}
|
|
11
|
+
*/
|
|
12
|
+
export function openDatabase(filename, mode) {
|
|
13
|
+
return new Promise((resolve, reject) => {
|
|
14
|
+
const db = new sqlite3.Database(filename, mode, (error) => {
|
|
15
|
+
if (error) reject(error);
|
|
16
|
+
else resolve(db);
|
|
17
|
+
});
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* @param {sqlite3.Database} db
|
|
23
|
+
* @param {string} sql
|
|
24
|
+
* @param {Array<any>} params
|
|
25
|
+
* @returns {Promise<{ changes: number, lastID: number }>}
|
|
26
|
+
*/
|
|
27
|
+
export function run(db, sql, params = []) {
|
|
28
|
+
return new Promise((resolve, reject) => {
|
|
29
|
+
db.run(sql, params, function onRun(error) {
|
|
30
|
+
if (error) reject(error);
|
|
31
|
+
else resolve({ changes: this.changes, lastID: this.lastID });
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @param {sqlite3.Database} db
|
|
38
|
+
* @param {string} sql
|
|
39
|
+
* @param {Array<any>} params
|
|
40
|
+
* @returns {Promise<Array<object>>}
|
|
41
|
+
*/
|
|
42
|
+
export function all(db, sql, params = []) {
|
|
43
|
+
return new Promise((resolve, reject) => {
|
|
44
|
+
db.all(sql, params, (error, rows) => {
|
|
45
|
+
if (error) reject(error);
|
|
46
|
+
else resolve(rows);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* @param {sqlite3.Database} db
|
|
53
|
+
* @param {string} sql
|
|
54
|
+
* @param {Array<any>} params
|
|
55
|
+
* @returns {Promise<object|undefined>}
|
|
56
|
+
*/
|
|
57
|
+
export function get(db, sql, params = []) {
|
|
58
|
+
return new Promise((resolve, reject) => {
|
|
59
|
+
db.get(sql, params, (error, row) => {
|
|
60
|
+
if (error) reject(error);
|
|
61
|
+
else resolve(row);
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @param {sqlite3.Database} db
|
|
68
|
+
* @param {string} sql
|
|
69
|
+
* @returns {Promise<void>}
|
|
70
|
+
*/
|
|
71
|
+
export function exec(db, sql) {
|
|
72
|
+
return new Promise((resolve, reject) => {
|
|
73
|
+
db.exec(sql, (error) => {
|
|
74
|
+
if (error) reject(error);
|
|
75
|
+
else resolve();
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* @param {sqlite3.Database|null} db
|
|
82
|
+
* @returns {Promise<void>}
|
|
83
|
+
*/
|
|
84
|
+
export function closeDatabase(db) {
|
|
85
|
+
if (!db) return Promise.resolve();
|
|
86
|
+
return new Promise((resolve, reject) => {
|
|
87
|
+
db.close((error) => {
|
|
88
|
+
if (error) reject(error);
|
|
89
|
+
else resolve();
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Maps a SQLite column type to a BigQuery column type.
|
|
7
|
+
* @param {string} type - The SQLite column type (e.g., 'TEXT', 'INTEGER', 'REAL').
|
|
8
|
+
* @returns {string} The BigQuery column type ('STRING', 'INT64', 'FLOAT64').
|
|
9
|
+
* @throws {Error} If the type is not supported.
|
|
10
|
+
*/
|
|
11
|
+
export function mapSqliteTypeToBigQuery(type) {
|
|
12
|
+
switch (type.toUpperCase()) {
|
|
13
|
+
case 'TEXT':
|
|
14
|
+
return 'STRING';
|
|
15
|
+
case 'INTEGER':
|
|
16
|
+
return 'INT64';
|
|
17
|
+
case 'REAL':
|
|
18
|
+
return 'FLOAT64';
|
|
19
|
+
default:
|
|
20
|
+
throw new Error(`Unsupported SQLite type: ${type}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Uploads entity rows to a BigQuery table, fully replacing the existing table data.
|
|
26
|
+
* The table will be created if it does not exist.
|
|
27
|
+
*
|
|
28
|
+
* @param {Object} options - Upload options.
|
|
29
|
+
* @param {import('@google-cloud/bigquery').BigQuery} options.bigquery - The BigQuery client instance.
|
|
30
|
+
* @param {string} options.datasetId - The name of the BigQuery dataset.
|
|
31
|
+
* @param {string} options.tableName - The name of the BigQuery table.
|
|
32
|
+
* @param {Array<{name: string, type: string}>} options.columns - The array of column definitions.
|
|
33
|
+
* @param {Array<Object>} options.rows - The array of row objects to upload.
|
|
34
|
+
* @returns {Promise<Object>} The completed BigQuery load job metadata.
|
|
35
|
+
*/
|
|
36
|
+
export async function uploadEntityTable({ bigquery, datasetId, tableName, columns, rows }) {
|
|
37
|
+
// Convert columns to BigQuery schema
|
|
38
|
+
const schema = columns.map(col => ({
|
|
39
|
+
name: col.name,
|
|
40
|
+
type: mapSqliteTypeToBigQuery(col.type),
|
|
41
|
+
}));
|
|
42
|
+
|
|
43
|
+
// Convert rows to NDJSON format
|
|
44
|
+
// We use join('\n') and handle empty arrays safely (it will return empty string)
|
|
45
|
+
const ndjson = rows.map(row => JSON.stringify(row)).join('\n');
|
|
46
|
+
|
|
47
|
+
// Write NDJSON to a temporary file
|
|
48
|
+
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'bq-upload-'));
|
|
49
|
+
const tempFilePath = path.join(tempDir, `${tableName}.json`);
|
|
50
|
+
|
|
51
|
+
try {
|
|
52
|
+
await fs.writeFile(tempFilePath, ndjson);
|
|
53
|
+
|
|
54
|
+
// Load into BigQuery
|
|
55
|
+
const dataset = bigquery.dataset(datasetId);
|
|
56
|
+
const table = dataset.table(tableName);
|
|
57
|
+
|
|
58
|
+
// table.load() polls the underlying job and resolves with its final
|
|
59
|
+
// metadata. It does not return the Job instance created by
|
|
60
|
+
// table.createLoadJob().
|
|
61
|
+
const [metadata] = await table.load(tempFilePath, {
|
|
62
|
+
sourceFormat: 'NEWLINE_DELIMITED_JSON',
|
|
63
|
+
writeDisposition: 'WRITE_TRUNCATE',
|
|
64
|
+
autodetect: false,
|
|
65
|
+
schema: { fields: schema },
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
// Check for load errors after the job completes
|
|
69
|
+
const status = metadata.status;
|
|
70
|
+
if (status.errors && status.errors.length > 0) {
|
|
71
|
+
throw new Error(
|
|
72
|
+
`BigQuery load job for ${tableName} failed: ${JSON.stringify(status.errors)}`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
return metadata;
|
|
77
|
+
} finally {
|
|
78
|
+
// Clean up temp file and directory
|
|
79
|
+
await fs.rm(tempDir, { recursive: true, force: true });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import assert from 'node:assert';
|
|
2
|
+
import test from 'node:test';
|
|
3
|
+
import { mapSqliteTypeToBigQuery, uploadEntityTable } from './upload_entities.js';
|
|
4
|
+
import fs from 'node:fs/promises';
|
|
5
|
+
|
|
6
|
+
test('mapSqliteTypeToBigQuery correctly maps types', () => {
|
|
7
|
+
assert.strictEqual(mapSqliteTypeToBigQuery('TEXT'), 'STRING');
|
|
8
|
+
assert.strictEqual(mapSqliteTypeToBigQuery('INTEGER'), 'INT64');
|
|
9
|
+
assert.strictEqual(mapSqliteTypeToBigQuery('REAL'), 'FLOAT64');
|
|
10
|
+
assert.strictEqual(mapSqliteTypeToBigQuery('text'), 'STRING');
|
|
11
|
+
|
|
12
|
+
assert.throws(() => mapSqliteTypeToBigQuery('BLOB'), /Unsupported SQLite type: BLOB/);
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test('uploadEntityTable prepares data and calls bigquery correctly', async () => {
|
|
16
|
+
const columns = [
|
|
17
|
+
{ name: 'id', type: 'INTEGER' },
|
|
18
|
+
{ name: 'name', type: 'TEXT' },
|
|
19
|
+
{ name: 'score', type: 'REAL' }
|
|
20
|
+
];
|
|
21
|
+
|
|
22
|
+
const rows = [
|
|
23
|
+
{ id: 1, name: 'Alice', score: 95.5 },
|
|
24
|
+
{ id: 2, name: 'Bob', score: 82.0 }
|
|
25
|
+
];
|
|
26
|
+
|
|
27
|
+
let loadedFilePath = null;
|
|
28
|
+
let loadedMetadata = null;
|
|
29
|
+
|
|
30
|
+
const mockTable = {
|
|
31
|
+
load: async (filePath, metadata) => {
|
|
32
|
+
loadedFilePath = filePath;
|
|
33
|
+
loadedMetadata = metadata;
|
|
34
|
+
|
|
35
|
+
// Verify file content during load
|
|
36
|
+
const content = await fs.readFile(filePath, 'utf-8');
|
|
37
|
+
assert.strictEqual(
|
|
38
|
+
content,
|
|
39
|
+
'{"id":1,"name":"Alice","score":95.5}\n{"id":2,"name":"Bob","score":82}'
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
return [{ status: { state: 'DONE', errors: [] } }];
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
const mockDataset = {
|
|
47
|
+
table: (tableName) => {
|
|
48
|
+
assert.strictEqual(tableName, 'test_table');
|
|
49
|
+
return mockTable;
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
const mockBigquery = {
|
|
54
|
+
dataset: (datasetId) => {
|
|
55
|
+
assert.strictEqual(datasetId, 'test_dataset');
|
|
56
|
+
return mockDataset;
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const metadata = await uploadEntityTable({
|
|
61
|
+
bigquery: mockBigquery,
|
|
62
|
+
datasetId: 'test_dataset',
|
|
63
|
+
tableName: 'test_table',
|
|
64
|
+
columns,
|
|
65
|
+
rows
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
assert.deepStrictEqual(metadata, { status: { state: 'DONE', errors: [] } });
|
|
69
|
+
|
|
70
|
+
assert.deepStrictEqual(loadedMetadata, {
|
|
71
|
+
sourceFormat: 'NEWLINE_DELIMITED_JSON',
|
|
72
|
+
writeDisposition: 'WRITE_TRUNCATE',
|
|
73
|
+
autodetect: false,
|
|
74
|
+
schema: {
|
|
75
|
+
fields: [
|
|
76
|
+
{ name: 'id', type: 'INT64' },
|
|
77
|
+
{ name: 'name', type: 'STRING' },
|
|
78
|
+
{ name: 'score', type: 'FLOAT64' }
|
|
79
|
+
]
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
// Verify temp file is cleaned up
|
|
84
|
+
const fileExists = await fs.access(loadedFilePath).then(() => true).catch(() => false);
|
|
85
|
+
assert.strictEqual(fileExists, false);
|
|
86
|
+
});
|