@tech4good/angular-schematics 21.4.0 → 21.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/package.json +1 -1
- package/src/entity/files/__name@dasherize__/__name@dasherize__.mock.ts.template +2 -1
- package/src/entity/files/__name@dasherize__/__name@dasherize__.model.ts.template +2 -1
- package/src/entity/index.js +57 -0
- package/src/entity/index.js.map +1 -1
- package/src/entity/schema.json +4 -0
- package/src/ng-add/files/.firebaserc.template +2 -1
- package/src/ng-add/files/.gitignore.template +2 -0
- package/src/ng-add/files/bigquery/queries/active_time_by_route.sql.template +34 -0
- package/src/ng-add/files/bigquery/scripts/export_entities.py.template +143 -0
- package/src/ng-add/files/bigquery/scripts/run_query.py.template +167 -0
- package/src/ng-add/files/bigquery/views/events_flat.sql.template +60 -0
- package/src/ng-add/files/bigquery/views/logs_latest.sql.template +34 -0
- package/src/ng-add/files/docs-infra/analytics-guide.md.template +170 -0
- package/src/ng-add/files/docs-infra/authentication-architecture.md.template +270 -0
- package/src/ng-add/files/docs-infra/infrastructure-guide.md.template +16 -10
- package/src/ng-add/files/firebase.json.template +6 -0
- package/src/ng-add/files/firebase.logs.json.template +5 -0
- package/src/ng-add/files/firestore.indexes.json.template +112 -0
- package/src/ng-add/files/firestore.logs.rules.spec.ts.template +100 -0
- package/src/ng-add/files/firestore.logs.rules.template +8 -0
- package/src/ng-add/files/firestore.rules.spec.ts.template +84 -15
- package/src/ng-add/files/firestore.rules.template +29 -17
- package/src/ng-add/files/functions/.gitignore.template +3 -0
- package/src/ng-add/files/functions/package.json.template +12 -0
- package/src/ng-add/files/functions/src/index.ts.template +44 -0
- package/src/ng-add/files/functions/tsconfig.json.template +13 -0
- package/src/ng-add/files/jest.config.ts.template +1 -1
- package/src/ng-add/files/package.json.template +7 -4
- package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.html.template +3 -1
- package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.spec.ts.template +6 -8
- package/src/ng-add/files/src/app/admin/admin-panel/admin-panel.component.ts.template +2 -0
- package/src/ng-add/files/src/app/app.component.html.template +2 -2
- package/src/ng-add/files/src/app/app.component.ts.template +2 -0
- package/src/ng-add/files/src/app/app.routes.ts.template +6 -0
- package/src/ng-add/files/src/app/auth-routing.spec.ts.template +9 -4
- package/src/ng-add/files/src/app/core/analytics/log.service.ts.template +26 -4
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.spec.ts.template +145 -2
- package/src/ng-add/files/src/app/core/analytics/time-analytics.service.ts.template +355 -149
- package/src/ng-add/files/src/app/core/auth/auth.service.ts.template +6 -1
- package/src/ng-add/files/src/app/core/auth/guards/require-admin.guard.spec.ts.template +38 -0
- package/src/ng-add/files/src/app/core/auth/guards/require-admin.guard.ts.template +3 -2
- package/src/ng-add/files/src/app/core/auth/guards/require-auth-and-consent.guard.ts.template +10 -3
- package/src/ng-add/files/src/app/core/firebase/cached-listeners.service.ts.template +11 -5
- package/src/ng-add/files/src/app/core/firebase/database.service.ts.template +34 -17
- package/src/ng-add/files/src/app/core/firebase/email-link-context.service.spec.ts.template +59 -0
- package/src/ng-add/files/src/app/core/firebase/email-link-context.service.ts.template +147 -0
- package/src/ng-add/files/src/app/core/firebase/firebase.mock.service.ts.template +104 -17
- package/src/ng-add/files/src/app/core/firebase/firebase.service.spec.ts.template +116 -0
- package/src/ng-add/files/src/app/core/firebase/firebase.service.ts.template +367 -70
- package/src/ng-add/files/src/app/core/store/app.store.ts.template +4 -1
- package/src/ng-add/files/src/app/core/store/auth/auth.mock.ts.template +52 -66
- package/src/ng-add/files/src/app/core/store/auth/auth.store.spec.ts.template +127 -0
- package/src/ng-add/files/src/app/core/store/auth/auth.store.ts.template +216 -147
- package/src/ng-add/files/src/app/core/store/user/user.mock.ts.template +2 -4
- package/src/ng-add/files/src/app/core/store/user/user.model.ts.template +1 -1
- package/src/ng-add/files/src/app/core/store/user-private/user-private.model.ts.template +1 -0
- package/src/ng-add/files/src/app/first-time/auth-action/auth-action.component.html.template +4 -0
- package/src/ng-add/files/src/app/first-time/auth-action/auth-action.component.scss.template +1 -0
- package/src/ng-add/files/src/app/first-time/auth-action/auth-action.component.ts.template +38 -0
- package/src/ng-add/files/src/app/first-time/consent/consent.component.html.template +172 -48
- package/src/ng-add/files/src/app/first-time/consent/consent.component.scss.template +31 -0
- package/src/ng-add/files/src/app/first-time/consent/consent.component.spec.ts.template +31 -3
- package/src/ng-add/files/src/app/first-time/consent/consent.component.ts.template +12 -11
- package/src/ng-add/files/src/app/first-time/consent/consent.guard.ts.template +5 -7
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.spec.ts.template +12 -1
- package/src/ng-add/files/src/app/first-time/early-access/early-access.component.ts.template +93 -48
- package/src/ng-add/files/src/app/first-time/early-access/early-access.guard.ts.template +12 -2
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.html.template +18 -0
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.scss.template +2 -0
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.spec.ts.template +40 -0
- package/src/ng-add/files/src/app/first-time/email-link/email-link.component.ts.template +76 -0
- package/src/ng-add/files/src/app/first-time/first-time-routing.spec.ts.template +14 -8
- package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.html.template +19 -7
- package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.scss.template +51 -0
- package/src/ng-add/files/src/app/first-time/landing/hero/hero.component.ts.template +29 -5
- package/src/ng-add/files/src/app/first-time/landing/landing.component.html.template +2 -1
- package/src/ng-add/files/src/app/first-time/landing/landing.component.scss.template +12 -8
- package/src/ng-add/files/src/app/first-time/landing/landing.component.spec.ts.template +31 -10
- package/src/ng-add/files/src/app/first-time/landing/landing.component.ts.template +6 -19
- package/src/ng-add/files/src/app/first-time/login/login.component.html.template +58 -6
- package/src/ng-add/files/src/app/first-time/login/login.component.scss.template +151 -22
- package/src/ng-add/files/src/app/first-time/login/login.component.spec.ts.template +18 -38
- package/src/ng-add/files/src/app/first-time/login/login.component.ts.template +59 -31
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.html.template +12 -0
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.scss.template +39 -0
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.spec.ts.template +68 -0
- package/src/ng-add/files/src/app/main/account-profile/account-profile.component.ts.template +20 -0
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.html.template +24 -0
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.scss.template +50 -0
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.spec.ts.template +65 -0
- package/src/ng-add/files/src/app/main/account-profile/delete-guest-card/delete-guest-card.component.ts.template +59 -0
- package/src/ng-add/files/src/app/main/account-profile/guest-deletion-dialog.component.ts.template +51 -0
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.html.template +50 -0
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.scss.template +137 -0
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.spec.ts.template +57 -0
- package/src/ng-add/files/src/app/main/account-profile/profile-details-card/profile-details-card.component.ts.template +59 -0
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.html.template +133 -0
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.scss.template +207 -0
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.spec.ts.template +74 -0
- package/src/ng-add/files/src/app/main/account-profile/sign-in-methods-card/sign-in-methods-card.component.ts.template +130 -0
- package/src/ng-add/files/src/app/main/home/home.component.ts.template +0 -5
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.html.template +15 -0
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.scss.template +79 -0
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.spec.ts.template +59 -0
- package/src/ng-add/files/src/app/shared/guest-warning-bar/guest-warning-bar.component.ts.template +44 -0
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.html.template +45 -15
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.scss.template +103 -19
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.spec.ts.template +73 -9
- package/src/ng-add/files/src/app/shared/navbar/navbar.component.ts.template +46 -18
- package/src/ng-add/files/src/environments/environment.prod.ts.template +1 -0
- package/src/ng-add/files/src/environments/environment.test.ts.template +1 -0
- package/src/ng-add/files/src/environments/environment.ts.template +1 -1
- package/src/ng-add/index_spec.d.ts +1 -0
- package/src/ng-add/index_spec.js +108 -0
- package/src/ng-add/index_spec.js.map +1 -0
- package/tsconfig.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,36 @@
|
|
|
1
|
+
## [21.6.0](https://github.com/tech4good-lab/angular-schematics/compare/v21.5.0...v21.6.0) (2026-07-28)
|
|
2
|
+
|
|
3
|
+
### Features
|
|
4
|
+
|
|
5
|
+
* **entity:** support foreign key fields and automated composite firestore index generation ([99ae303](https://github.com/tech4good-lab/angular-schematics/commit/99ae303cd6eee6fae24fda7c994a178e0aa5be11))
|
|
6
|
+
* **ng-add:** add account profile, guest banner, and login UI components ([174d9a5](https://github.com/tech4good-lab/angular-schematics/commit/174d9a5f6d92a14e359e44193f98289bb30d9dac))
|
|
7
|
+
* **ng-add:** add cloud functions infrastructure and security rules ([9c65a61](https://github.com/tech4good-lab/angular-schematics/commit/9c65a6119880243c5f624ddb312ed61e287dbe47))
|
|
8
|
+
* **ng-add:** enhance auth services, stores, and guards ([3396eee](https://github.com/tech4good-lab/angular-schematics/commit/3396eee629a85865fd66f9965825aca4e564956a))
|
|
9
|
+
* **ng-add:** update auth token initialization and test environments ([9206c6b](https://github.com/tech4good-lab/angular-schematics/commit/9206c6b5e591d3aa77de82a37225094b724f1955))
|
|
10
|
+
* **ng-add:** update default firestore indexes and security rule state transitions ([28e851f](https://github.com/tech4good-lab/angular-schematics/commit/28e851fd5cdf9d9c441ceeebcd11080c87f69261))
|
|
11
|
+
* **ng-add:** update landing, hero, and early access component layouts ([78b8918](https://github.com/tech4good-lab/angular-schematics/commit/78b891855a5a0340384e339fb2a7e56bb12f6fd3))
|
|
12
|
+
* **shared:** align navbar component layout and avatar fallback ([9f80c5f](https://github.com/tech4good-lab/angular-schematics/commit/9f80c5f296ff5c2d0ba3245d5656031ead804683))
|
|
13
|
+
|
|
14
|
+
### Build System
|
|
15
|
+
|
|
16
|
+
* **ng-add:** bump template dependency for release ([5e0b098](https://github.com/tech4good-lab/angular-schematics/commit/5e0b0981629d0adddffc5b7d66e955d1cd9ae86e))
|
|
17
|
+
## [21.5.0](https://github.com/tech4good-lab/angular-schematics/compare/v21.4.0...v21.5.0) (2026-07-22)
|
|
18
|
+
|
|
19
|
+
### Features
|
|
20
|
+
|
|
21
|
+
* **ng-add:** add Firestore logging environment and security rules templates ([d5151c0](https://github.com/tech4good-lab/angular-schematics/commit/d5151c0f3ac63d5cd935cdf07a9fd970ac27f16a))
|
|
22
|
+
* **ng-add:** scaffold BigQuery views, queries, guide, and CLI scripts ([b4c2b7b](https://github.com/tech4good-lab/angular-schematics/commit/b4c2b7b44d6144cb9bf07be43431ef288d460206))
|
|
23
|
+
* **ng-add:** support object serialization in telemetry interactions ([ac14a3a](https://github.com/tech4good-lab/angular-schematics/commit/ac14a3ac290b12528eb52bc1ecb4233b93f3bc9a))
|
|
24
|
+
* **ng-add:** support revisitable consent page ([602ee5e](https://github.com/tech4good-lab/angular-schematics/commit/602ee5eaa25249e324b70aea2dce75d1c3a2b67b))
|
|
25
|
+
* **ng-add:** update consent component template and legal text ([bdf6926](https://github.com/tech4good-lab/angular-schematics/commit/bdf69262974346530f5137535042f38b4b1c54e9))
|
|
26
|
+
|
|
27
|
+
### Code Refactoring
|
|
28
|
+
|
|
29
|
+
* **ng-add:** update core time analytics service and analytics guide templates ([7781f43](https://github.com/tech4good-lab/angular-schematics/commit/7781f43ba79f20e98b3fe73bea2854f1bdd8e89c))
|
|
30
|
+
|
|
31
|
+
### Build System
|
|
32
|
+
|
|
33
|
+
* **ng-add:** bump template dependency for release ([5b644cd](https://github.com/tech4good-lab/angular-schematics/commit/5b644cd9990e0eb8b9c636a35ab1be8c49877464))
|
|
1
34
|
## [21.4.0](https://github.com/tech4good-lab/angular-schematics/compare/v21.3.0...v21.4.0) (2026-07-08)
|
|
2
35
|
|
|
3
36
|
### Features
|
package/package.json
CHANGED
|
@@ -14,7 +14,8 @@ export const <%= classify(name) %>MockDB = signalStore(
|
|
|
14
14
|
*/
|
|
15
15
|
export function createMock<%= classify(name) %>(overrides?: Partial<<%= classify(name) %>>): <%= classify(name) %> {
|
|
16
16
|
return {
|
|
17
|
-
__id: '<%= dasherize(name) %>-default'
|
|
17
|
+
__id: '<%= dasherize(name) %>-default',<% if (parsedForeignKeys && parsedForeignKeys.length > 0) { parsedForeignKeys.forEach(function(fk) { %>
|
|
18
|
+
__<%= fk %>: '<%= fk %>-default',<% }); } %>
|
|
18
19
|
_createdAt: Timestamp.now(),
|
|
19
20
|
_updatedAt: Timestamp.now(),
|
|
20
21
|
_deleted: false,
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { Timestamp } from '@angular/fire/firestore';
|
|
2
2
|
|
|
3
3
|
export interface <%= classify(name) %> {
|
|
4
|
-
__id: string
|
|
4
|
+
__id: string;<% if (parsedForeignKeys && parsedForeignKeys.length > 0) { parsedForeignKeys.forEach(function(fk) { %>
|
|
5
|
+
__<%= fk %>: string;<% }); } %>
|
|
5
6
|
_createdAt?: Timestamp;
|
|
6
7
|
_updatedAt?: Timestamp;
|
|
7
8
|
_deleted?: boolean;
|
package/src/entity/index.js
CHANGED
|
@@ -88,6 +88,47 @@ function addInserts(options) {
|
|
|
88
88
|
return host;
|
|
89
89
|
};
|
|
90
90
|
}
|
|
91
|
+
function addFirestoreIndex(options) {
|
|
92
|
+
return (host, context) => {
|
|
93
|
+
const parsedForeignKeys = options.parsedForeignKeys || [];
|
|
94
|
+
if (parsedForeignKeys.length === 0) {
|
|
95
|
+
return host;
|
|
96
|
+
}
|
|
97
|
+
const indexPath = '/firestore.indexes.json';
|
|
98
|
+
if (!host.exists(indexPath)) {
|
|
99
|
+
context.logger.warn('firestore.indexes.json not found — skipping index generation.');
|
|
100
|
+
return host;
|
|
101
|
+
}
|
|
102
|
+
const camelizedName = core_1.strings.camelize(options.name);
|
|
103
|
+
const collectionGroup = camelizedName + 's';
|
|
104
|
+
// Build the composite index fields: foreign keys + _deleted + _updatedAt + __name__
|
|
105
|
+
const fields = [
|
|
106
|
+
...parsedForeignKeys.map((fk) => ({
|
|
107
|
+
fieldPath: `__${fk}`,
|
|
108
|
+
order: 'ASCENDING',
|
|
109
|
+
})),
|
|
110
|
+
{ fieldPath: '_deleted', order: 'ASCENDING' },
|
|
111
|
+
{ fieldPath: '_updatedAt', order: 'ASCENDING' },
|
|
112
|
+
{ fieldPath: '__name__', order: 'ASCENDING' },
|
|
113
|
+
];
|
|
114
|
+
const newIndex = {
|
|
115
|
+
collectionGroup,
|
|
116
|
+
queryScope: 'COLLECTION',
|
|
117
|
+
fields,
|
|
118
|
+
density: 'SPARSE_ALL',
|
|
119
|
+
};
|
|
120
|
+
const indexContent = host.read(indexPath);
|
|
121
|
+
if (!indexContent) {
|
|
122
|
+
context.logger.warn('Could not read firestore.indexes.json — skipping index generation.');
|
|
123
|
+
return host;
|
|
124
|
+
}
|
|
125
|
+
const indexJson = JSON.parse(indexContent.toString());
|
|
126
|
+
indexJson.indexes.push(newIndex);
|
|
127
|
+
host.overwrite(indexPath, JSON.stringify(indexJson, null, 2) + '\n');
|
|
128
|
+
context.logger.info(`Added composite Firestore index for ${collectionGroup} with foreign keys: ${parsedForeignKeys.map(fk => '__' + fk).join(', ')}`);
|
|
129
|
+
return host;
|
|
130
|
+
};
|
|
131
|
+
}
|
|
91
132
|
function entity(options) {
|
|
92
133
|
return async (tree, context) => {
|
|
93
134
|
const workspace = await (0, workspace_1.getWorkspace)(tree);
|
|
@@ -123,6 +164,11 @@ function entity(options) {
|
|
|
123
164
|
const parsedPath = (0, parse_name_1.parseName)(options.path, options.name);
|
|
124
165
|
options.name = parsedPath.name;
|
|
125
166
|
options.path = parsedPath.path;
|
|
167
|
+
// Parse foreign keys from comma-separated string
|
|
168
|
+
const parsedForeignKeys = options.foreignKeys
|
|
169
|
+
? options.foreignKeys.split(',').map((k) => k.trim()).filter((k) => k.length > 0)
|
|
170
|
+
: [];
|
|
171
|
+
options.parsedForeignKeys = parsedForeignKeys;
|
|
126
172
|
const templateSource = (0, schematics_1.apply)((0, schematics_1.url)('./files'), [
|
|
127
173
|
(0, schematics_1.applyTemplates)({
|
|
128
174
|
...core_1.strings,
|
|
@@ -130,8 +176,19 @@ function entity(options) {
|
|
|
130
176
|
}),
|
|
131
177
|
(0, schematics_1.move)(parsedPath.path),
|
|
132
178
|
]);
|
|
179
|
+
// Post-generation reminder about foreign keys and indexes
|
|
180
|
+
if (parsedForeignKeys.length > 0) {
|
|
181
|
+
context.logger.info(`\n📋 REMINDER: Don't forget to deploy the updated Firestore indexes:\n` +
|
|
182
|
+
` firebase deploy --only firestore:indexes\n`);
|
|
183
|
+
}
|
|
184
|
+
else {
|
|
185
|
+
context.logger.info(`\n📋 REMINDER: If your entity has foreign key fields (e.g., __userId, __collectionId),\n` +
|
|
186
|
+
` re-generate with --foreign-keys to create the required composite Firestore indexes:\n` +
|
|
187
|
+
` ng generate @tech4good/angular-schematics:entity ${options.name} --foreign-keys=userId,collectionId\n`);
|
|
188
|
+
}
|
|
133
189
|
return (0, schematics_1.chain)([
|
|
134
190
|
addInserts(options),
|
|
191
|
+
addFirestoreIndex(options),
|
|
135
192
|
(0, schematics_1.mergeWith)(templateSource)
|
|
136
193
|
]);
|
|
137
194
|
};
|
package/src/entity/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":";;AA4KA,wBA4EC;AAxPD,2DAIoC;AAEpC,+DAA0E;AAC1E,yEAA4E;AAC5E,uEAAmE;AACnE,qEAAuF;AACvF,+CAA+C;AAE/C,iEAAuF;AAEvF,SAAS,UAAU,CAAC,OAAY;IAC9B,OAAO,CAAC,IAAU,EAAE,OAAyB,EAAE,EAAE;QAE/C,MAAM,cAAc,GAAG,cAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACtD,MAAM,aAAa,GAAG,cAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,cAAc,GAAG,cAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,WAAW,GAAG,cAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACjE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,cAAc,KAAK,aAAa,EAAE,CAAC,CAAC;QAElE,uBAAuB;QACvB,MAAM,YAAY,GAAG,kCAAkC,CAAC;QACxD,MAAM,YAAY,GAAG,IAAA,gCAAa,EAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QACvD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,YAAY,EAAE,CAAC,CAAC;QAErD,MAAM,eAAe,GACnB,IAAI,OAAO,CAAC,IAAI,GAAG;YACnB,cAAc,GAAG,GAAG;YACpB,cAAc,GAAG,QAAQ,CAAC;QAC5B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,eAAe,EAAE,CAAC,CAAC;QAC3D,MAAM,oBAAoB,GAAG,IAAA,+BAAiB,EAAC,YAAY,EAAE,eAAe,CAAC,CAAC;QAC9E,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,oBAAoB,EAAE,CAAC,CAAC;QAEhE,IAAI,uBAAuB,GAAG,IAAA,+BAAY,EACxC,YAAY,EACZ,YAAY,EACZ;YACE,kBAAkB;YAClB,yBAAyB;SAC1B,EACD;YACE,YAAY,cAAc,YAAY,oBAAoB,IAAI;YAC9D,KAAK,cAAc,IAAI;SACxB,CACF,CAAC;QAEF,mCAAmC;QACnC,MAAM,UAAU,GAAG,iDAAiD,CAAC;QACrE,MAAM,UAAU,GAAG,IAAA,gCAAa,EAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEnD,MAAM,cAAc,GAClB,IAAI,OAAO,CAAC,IAAI,GAAG;YACnB,cAAc,GAAG,GAAG;YACpB,cAAc,GAAG,OAAO,CAAC;QAC3B,MAAM,kBAAkB,GAAG,IAAA,+BAAiB,EAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QACzE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;QAEjD,IAAI,qBAAqB,GAAG,IAAA,+BAAY,EACtC,UAAU,EACV,UAAU,EACV;YACE,yBAAyB;YACzB,iBAAiB;YACjB,uBAAuB;YACvB,uCAAuC;YACvC,MAAM;YACN,oDAAoD;SACrD,EACD;YACE,YAAY,cAAc,WAAW,WAAW,eAAe,kBAAkB,IAAI;YACrF,QAAQ,aAAa,cAAc,cAAc,UAAU;YAC3D,gBAAgB,aAAa,YAAY,WAAW,OAAO;YAC3D,KAAK,aAAa,IAAI;YACtB,KAAK,aAAa,YAAY;YAC9B,eAAe,aAAa,kBAAkB,aAAa,YAAY,aAAa,KAAK;SAC1F,CACF,CAAC;QAEF,0BAA0B;QAC1B,MAAM,SAAS,GAAG,kBAAkB,CAAC;QACrC,IAAI,kBAAkB,GAAa,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,IAAA,gCAAa,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YACjD,IAAI,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC9C,kBAAkB,CAAC,IAAI,CAAC,IAAA,+BAAY,EAClC,SAAS,EACT,SAAS,EACT,qBAAqB,EACrB,UAAU,cAAc,2EAA2E;oBACnG,cAAc,aAAa,eAAe;oBAC1C,qDAAqD;oBACrD,eAAe,CAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACjD,6EAA6E;QAC7E,MAAM,WAAW,GAAyB;YACxC,CAAC,YAAY,EAAE,uBAAuB,CAAC;YACvC,CAAC,UAAU,EAAE,qBAAqB,CAAC;SACpC,CAAC;QACF,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAC;QACpD,CAAC;QAED,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE;YAChD,MAAM,mBAAmB,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YACvD,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,IAAI,MAAM,YAAY,qBAAY,EAAE,CAAC;oBACnC,mBAAmB,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC3D,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC,CAAA;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAY;IACrC,OAAO,CAAC,IAAU,EAAE,OAAyB,EAAE,EAAE;QAC/C,MAAM,iBAAiB,GAAa,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAC;QACpE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GAAG,yBAAyB,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;YACrF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,aAAa,GAAG,cAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,eAAe,GAAG,aAAa,GAAG,GAAG,CAAC;QAE5C,oFAAoF;QACpF,MAAM,MAAM,GAA2C;YACrD,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,EAAU,EAAE,EAAE,CAAC,CAAC;gBACxC,SAAS,EAAE,KAAK,EAAE,EAAE;gBACpB,KAAK,EAAE,WAAW;aACnB,CAAC,CAAC;YACH,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE;YAC7C,EAAE,SAAS,EAAE,YAAY,EAAE,KAAK,EAAE,WAAW,EAAE;YAC/C,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE;SAC9C,CAAC;QAEF,MAAM,QAAQ,GAAG;YACf,eAAe;YACf,UAAU,EAAE,YAAY;YACxB,MAAM;YACN,OAAO,EAAE,YAAY;SACtB,CAAC;QAEF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;YAC1F,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QAErE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,eAAe,uBAAuB,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtJ,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,MAAM,CAAC,OAAY;IACjC,OAAO,KAAK,EAAE,IAAU,EAAE,OAAyB,EAAE,EAAE;QAErD,MAAM,SAAS,GAAG,MAAM,IAAA,wBAAY,EAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,OAAiB,CAAC,CAAC;QAElE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,gCAAmB,CAAC,YAAY,OAAO,CAAC,OAAO,mBAAmB,CAAC,CAAC;QAChF,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,GAAG,IAAA,4BAAgB,EAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;YACjE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,2HAA2H,CAAC,CAAC;YAEjJ,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;gBACzB,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;gBACjE,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBACjG,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;oBACpC;wBACE,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,UAAU;wBAChB,OAAO,EAAE,gEAAgE;wBACzE,OAAO,EAAE,IAAI;qBACd;iBACF,CAAC,CAAC;gBACH,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;YAC9B,CAAC;YAED,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,GAAG,oBAAoB,CAAC;YACtC,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,IAAA,sBAAS,EAAC,OAAO,CAAC,IAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;QAC/B,OAAO,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;QAE/B,iDAAiD;QACjD,MAAM,iBAAiB,GAAa,OAAO,CAAC,WAAW;YACrD,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACjG,CAAC,CAAC,EAAE,CAAC;QACP,OAAO,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE9C,MAAM,cAAc,GAAG,IAAA,kBAAK,EAAC,IAAA,gBAAG,EAAC,SAAS,CAAC,EAAE;YAC3C,IAAA,2BAAc,EAAC;gBACb,GAAG,cAAO;gBACV,GAAG,OAAO;aACX,CAAC;YACF,IAAA,iBAAI,EAAC,UAAU,CAAC,IAAI,CAAC;SACtB,CAAC,CAAC;QAEH,0DAA0D;QAC1D,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,MAAM,CAAC,IAAI,CACjB,wEAAwE;gBACxE,+CAA+C,CAChD,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,IAAI,CACjB,0FAA0F;gBAC1F,0FAA0F;gBAC1F,uDAAuD,OAAO,CAAC,IAAI,uCAAuC,CAC3G,CAAC;QACJ,CAAC;QAED,OAAO,IAAA,kBAAK,EAAC;YACX,UAAU,CAAC,OAAO,CAAC;YACnB,iBAAiB,CAAC,OAAO,CAAC;YAC1B,IAAA,sBAAS,EAAC,cAAc,CAAC;SAC1B,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC"}
|
package/src/entity/schema.json
CHANGED
|
@@ -33,6 +33,10 @@
|
|
|
33
33
|
"index": 0
|
|
34
34
|
},
|
|
35
35
|
"x-prompt": "What name would you like to use for the entity?"
|
|
36
|
+
},
|
|
37
|
+
"foreignKeys": {
|
|
38
|
+
"type": "string",
|
|
39
|
+
"description": "Comma-separated list of foreign key names (without __ prefix). E.g., 'userId,collectionId'. These will be added to the model interface as __fieldName: string fields, and composite Firestore indexes will be generated for efficient querying."
|
|
36
40
|
}
|
|
37
41
|
},
|
|
38
42
|
"required": [
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
"projects": {
|
|
3
3
|
"default": "<%= dasherize(project) %>-dev",
|
|
4
4
|
"dev": "<%= dasherize(project) %>-dev",
|
|
5
|
-
"prod": "<%= dasherize(project) %>-prod"
|
|
5
|
+
"prod": "<%= dasherize(project) %>-prod",
|
|
6
|
+
"log": "<%= dasherize(project) %>-log"
|
|
6
7
|
},
|
|
7
8
|
"targets": {},
|
|
8
9
|
"etags": {}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
-- Query: active_time_by_route
|
|
2
|
+
-- Description: Analyzes the distribution of active engagement time (in seconds) spent
|
|
3
|
+
-- on different routes. Computes total visits, mean active seconds, and percentiles
|
|
4
|
+
-- (Min, Q1, Median, Q3, Max) for each route.
|
|
5
|
+
|
|
6
|
+
WITH route_visit_times AS (
|
|
7
|
+
SELECT
|
|
8
|
+
route,
|
|
9
|
+
-- Convert milliseconds to seconds
|
|
10
|
+
total_active_ms / 1000.0 AS active_seconds
|
|
11
|
+
FROM
|
|
12
|
+
`analytics.events_flat`
|
|
13
|
+
WHERE
|
|
14
|
+
event_type = 'route_visit'
|
|
15
|
+
AND total_active_ms > 0
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
SELECT
|
|
19
|
+
route,
|
|
20
|
+
COUNT(*) AS total_visits,
|
|
21
|
+
ROUND(AVG(active_seconds), 2) AS mean_active_seconds,
|
|
22
|
+
|
|
23
|
+
-- APPROX_QUANTILES returns a 5-element array: [Min, 25th (Q1), 50th (Median), 75th (Q3), Max]
|
|
24
|
+
ROUND(APPROX_QUANTILES(active_seconds, 4)[OFFSET(0)], 2) AS min_active_seconds,
|
|
25
|
+
ROUND(APPROX_QUANTILES(active_seconds, 4)[OFFSET(1)], 2) AS q1_active_seconds,
|
|
26
|
+
ROUND(APPROX_QUANTILES(active_seconds, 4)[OFFSET(2)], 2) AS median_active_seconds,
|
|
27
|
+
ROUND(APPROX_QUANTILES(active_seconds, 4)[OFFSET(3)], 2) AS q3_active_seconds,
|
|
28
|
+
ROUND(APPROX_QUANTILES(active_seconds, 4)[OFFSET(4)], 2) AS max_active_seconds
|
|
29
|
+
FROM
|
|
30
|
+
route_visit_times
|
|
31
|
+
GROUP BY
|
|
32
|
+
route
|
|
33
|
+
ORDER BY
|
|
34
|
+
total_visits DESC;
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Firestore Entity Exporter
|
|
4
|
+
|
|
5
|
+
This script exports any specified Firestore collection to a local JSON or CSV file.
|
|
6
|
+
Useful for pulling low-volume, PII-sensitive collections (like user responses)
|
|
7
|
+
on-demand to run Jupyter notebook analyses without storing them persistently in BigQuery.
|
|
8
|
+
|
|
9
|
+
Usage:
|
|
10
|
+
# Export to JSON
|
|
11
|
+
python bigquery/scripts/export_entities.py --collection <collection-name> --output data.json
|
|
12
|
+
|
|
13
|
+
# Export to CSV
|
|
14
|
+
python bigquery/scripts/export_entities.py --collection <collection-name> --output data.csv
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import csv
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import sys
|
|
22
|
+
from datetime import datetime
|
|
23
|
+
|
|
24
|
+
try:
|
|
25
|
+
import firebase_admin
|
|
26
|
+
from firebase_admin import credentials, firestore
|
|
27
|
+
from google.api_core.datetime_helpers import DatetimeWithNanoseconds
|
|
28
|
+
except ImportError:
|
|
29
|
+
print("Error: Required packages missing. Please install dependencies:")
|
|
30
|
+
print(" pip install firebase-admin google-api-core")
|
|
31
|
+
sys.exit(1)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def parse_value(val):
|
|
35
|
+
"""Recursively converts Firestore Timestamp objects and other non-JSON-serializable types."""
|
|
36
|
+
if isinstance(val, DatetimeWithNanoseconds):
|
|
37
|
+
return val.isoformat()
|
|
38
|
+
elif isinstance(val, datetime):
|
|
39
|
+
return val.isoformat()
|
|
40
|
+
elif isinstance(val, dict):
|
|
41
|
+
return {k: parse_value(v) for k, v in val.items()}
|
|
42
|
+
elif isinstance(val, list):
|
|
43
|
+
return [parse_value(x) for x in val]
|
|
44
|
+
return val
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def main():
|
|
48
|
+
parser = argparse.ArgumentParser(description="Export Firestore collections to JSON or CSV.")
|
|
49
|
+
parser.add_argument(
|
|
50
|
+
"--collection",
|
|
51
|
+
required=True,
|
|
52
|
+
help="Name of the Firestore collection to export",
|
|
53
|
+
)
|
|
54
|
+
parser.add_argument(
|
|
55
|
+
"--output",
|
|
56
|
+
required=True,
|
|
57
|
+
help="Local file path for the export (must end in .json or .csv)",
|
|
58
|
+
)
|
|
59
|
+
parser.add_argument(
|
|
60
|
+
"--credentials",
|
|
61
|
+
help="Path to the Firebase service account key JSON file. "
|
|
62
|
+
"If omitted, falls back to GOOGLE_APPLICATION_CREDENTIALS env var.",
|
|
63
|
+
)
|
|
64
|
+
args = parser.parse_args()
|
|
65
|
+
|
|
66
|
+
# 1. Initialize Firebase App
|
|
67
|
+
if not firebase_admin._apps:
|
|
68
|
+
try:
|
|
69
|
+
if args.credentials:
|
|
70
|
+
cred = credentials.Certificate(args.credentials)
|
|
71
|
+
firebase_admin.initialize_app(cred)
|
|
72
|
+
elif os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):
|
|
73
|
+
cred = credentials.Certificate(os.environ["GOOGLE_APPLICATION_CREDENTIALS"])
|
|
74
|
+
firebase_admin.initialize_app(cred)
|
|
75
|
+
else:
|
|
76
|
+
# Attempt to load Application Default Credentials
|
|
77
|
+
firebase_admin.initialize_app()
|
|
78
|
+
except Exception as e:
|
|
79
|
+
print("Authentication Error: Could not initialize Firebase Admin SDK.")
|
|
80
|
+
print("Please either:")
|
|
81
|
+
print(" 1. Download a service account JSON file from Firebase Console -> Project Settings -> Service accounts.")
|
|
82
|
+
print(" 2. Pass it using --credentials /path/to/key.json")
|
|
83
|
+
print(" 3. Or run: export GOOGLE_APPLICATION_CREDENTIALS='/path/to/key.json'")
|
|
84
|
+
print(f"Details: {e}")
|
|
85
|
+
sys.exit(1)
|
|
86
|
+
|
|
87
|
+
db = firestore.client()
|
|
88
|
+
collection_ref = db.collection(args.collection)
|
|
89
|
+
|
|
90
|
+
print(f"Fetching documents from collection '{args.collection}'...")
|
|
91
|
+
docs = list(collection_ref.stream())
|
|
92
|
+
print(f"Retrieved {len(docs)} documents.")
|
|
93
|
+
|
|
94
|
+
if not docs:
|
|
95
|
+
print("No documents found. Exiting.")
|
|
96
|
+
return
|
|
97
|
+
|
|
98
|
+
# 2. Extract and format data
|
|
99
|
+
flat_data = []
|
|
100
|
+
headers = set()
|
|
101
|
+
|
|
102
|
+
for doc in docs:
|
|
103
|
+
doc_dict = doc.to_dict()
|
|
104
|
+
doc_dict["__id"] = doc.id # Include primary document ID
|
|
105
|
+
parsed_dict = parse_value(doc_dict)
|
|
106
|
+
flat_data.append(parsed_dict)
|
|
107
|
+
headers.update(parsed_dict.keys())
|
|
108
|
+
|
|
109
|
+
# Sort headers to ensure consistent column ordering in CSV, placing __id first
|
|
110
|
+
headers = sorted(list(headers))
|
|
111
|
+
if "__id" in headers:
|
|
112
|
+
headers.remove("__id")
|
|
113
|
+
headers.insert(0, "__id")
|
|
114
|
+
|
|
115
|
+
# 3. Write output
|
|
116
|
+
output_path = args.output
|
|
117
|
+
ext = os.path.splitext(output_path)[1].lower()
|
|
118
|
+
|
|
119
|
+
if ext == ".json":
|
|
120
|
+
with open(output_path, "w", encoding="utf-8") as f:
|
|
121
|
+
json.dump(flat_data, f, indent=2, ensure_ascii=False)
|
|
122
|
+
print(f"Successfully exported data to JSON: {output_path}")
|
|
123
|
+
|
|
124
|
+
elif ext == ".csv":
|
|
125
|
+
with open(output_path, "w", encoding="utf-8", newline="") as f:
|
|
126
|
+
writer = csv.DictWriter(f, fieldnames=headers, extrasaction="ignore")
|
|
127
|
+
writer.writeheader()
|
|
128
|
+
for row in flat_data:
|
|
129
|
+
csv_row = {}
|
|
130
|
+
for k, v in row.items():
|
|
131
|
+
if isinstance(v, (dict, list)):
|
|
132
|
+
csv_row[k] = json.dumps(v, ensure_ascii=False)
|
|
133
|
+
else:
|
|
134
|
+
csv_row[k] = v
|
|
135
|
+
writer.writerow(csv_row)
|
|
136
|
+
print(f"Successfully exported data to CSV: {output_path}")
|
|
137
|
+
else:
|
|
138
|
+
print(f"Error: Unknown output format '{ext}'. Must be .json or .csv.")
|
|
139
|
+
sys.exit(1)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
if __name__ == "__main__":
|
|
143
|
+
main()
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
BigQuery Query Runner
|
|
4
|
+
|
|
5
|
+
Runs a specified SQL query file against Google BigQuery and exports the results
|
|
6
|
+
to JSON, CSV, or a local SQLite database.
|
|
7
|
+
|
|
8
|
+
Usage:
|
|
9
|
+
# Export to JSON
|
|
10
|
+
python bigquery/scripts/run_query.py --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output output.json
|
|
11
|
+
|
|
12
|
+
# Export to CSV
|
|
13
|
+
python bigquery/scripts/run_query.py --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output output.csv
|
|
14
|
+
|
|
15
|
+
# Export to SQLite
|
|
16
|
+
python bigquery/scripts/run_query.py --project <your-gcp-project-id> --query bigquery/queries/active_time_by_route.sql --output metrics.db --table active_time
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import argparse
|
|
20
|
+
import csv
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import sqlite3
|
|
24
|
+
import sys
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
from google.cloud import bigquery
|
|
28
|
+
except ImportError:
|
|
29
|
+
print("Error: Required package 'google-cloud-bigquery' missing. Please install it:")
|
|
30
|
+
print(" pip install google-cloud-bigquery")
|
|
31
|
+
sys.exit(1)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def main():
|
|
35
|
+
parser = argparse.ArgumentParser(description="Run BigQuery SQL queries and export results.")
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"--query",
|
|
38
|
+
required=True,
|
|
39
|
+
help="Path to the SQL file to run (e.g., bigquery/queries/active_time_by_route.sql)",
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"--output",
|
|
43
|
+
required=True,
|
|
44
|
+
help="Local file path for the export (must end in .json, .csv, or .db/.sqlite)",
|
|
45
|
+
)
|
|
46
|
+
parser.add_argument(
|
|
47
|
+
"--project",
|
|
48
|
+
required=True,
|
|
49
|
+
help="GCP Project ID to run the query against (e.g., <your-gcp-project-id>)",
|
|
50
|
+
)
|
|
51
|
+
parser.add_argument(
|
|
52
|
+
"--table",
|
|
53
|
+
help="Target table name when exporting to a SQLite database. Required if output is SQLite.",
|
|
54
|
+
)
|
|
55
|
+
parser.add_argument(
|
|
56
|
+
"--credentials",
|
|
57
|
+
help="Path to the Google service account key JSON file. "
|
|
58
|
+
"If omitted, falls back to GOOGLE_APPLICATION_CREDENTIALS env var.",
|
|
59
|
+
)
|
|
60
|
+
args = parser.parse_args()
|
|
61
|
+
|
|
62
|
+
# 1. Validate inputs
|
|
63
|
+
if not os.path.exists(args.query):
|
|
64
|
+
print(f"Error: Query file not found at {args.query}")
|
|
65
|
+
sys.exit(1)
|
|
66
|
+
|
|
67
|
+
output_ext = os.path.splitext(args.output)[1].lower()
|
|
68
|
+
is_sqlite = output_ext in [".db", ".sqlite", ".sqlite3"]
|
|
69
|
+
if is_sqlite and not args.table:
|
|
70
|
+
print("Error: --table argument is required when exporting to a SQLite database.")
|
|
71
|
+
sys.exit(1)
|
|
72
|
+
|
|
73
|
+
with open(args.query, "r", encoding="utf-8") as f:
|
|
74
|
+
query_string = f.read()
|
|
75
|
+
|
|
76
|
+
# 2. Initialize BigQuery Client
|
|
77
|
+
try:
|
|
78
|
+
if args.credentials:
|
|
79
|
+
client = bigquery.Client.from_service_account_json(args.credentials, project=args.project)
|
|
80
|
+
else:
|
|
81
|
+
client = bigquery.Client(project=args.project)
|
|
82
|
+
except Exception as e:
|
|
83
|
+
print("Authentication Error: Could not initialize BigQuery Client.")
|
|
84
|
+
print("Please check your credentials or run:")
|
|
85
|
+
print(" export GOOGLE_APPLICATION_CREDENTIALS='/path/to/key.json'")
|
|
86
|
+
print(f"Details: {e}")
|
|
87
|
+
sys.exit(1)
|
|
88
|
+
|
|
89
|
+
# 3. Execute Query
|
|
90
|
+
print(f"Executing query '{args.query}' in project '{args.project}'...")
|
|
91
|
+
try:
|
|
92
|
+
query_job = client.query(query_string)
|
|
93
|
+
results = query_job.result() # Waits for query to complete
|
|
94
|
+
print("Query completed successfully. Fetching rows...")
|
|
95
|
+
except Exception as e:
|
|
96
|
+
print(f"Query Execution Error: {e}")
|
|
97
|
+
sys.exit(1)
|
|
98
|
+
|
|
99
|
+
# Convert rows to a list of dicts for export
|
|
100
|
+
rows = [dict(row) for row in results]
|
|
101
|
+
if not rows:
|
|
102
|
+
print("Query returned 0 rows. Exiting.")
|
|
103
|
+
return
|
|
104
|
+
|
|
105
|
+
# 4. Write Output
|
|
106
|
+
if output_ext == ".json":
|
|
107
|
+
with open(args.output, "w", encoding="utf-8") as f:
|
|
108
|
+
json.dump(rows, f, indent=2, default=str)
|
|
109
|
+
print(f"Successfully exported {len(rows)} rows to JSON: {args.output}")
|
|
110
|
+
|
|
111
|
+
elif output_ext == ".csv":
|
|
112
|
+
headers = list(rows[0].keys())
|
|
113
|
+
with open(args.output, "w", encoding="utf-8", newline="") as f:
|
|
114
|
+
writer = csv.DictWriter(f, fieldnames=headers)
|
|
115
|
+
writer.writeheader()
|
|
116
|
+
for row in rows:
|
|
117
|
+
# Stringify nested structures
|
|
118
|
+
row_clean = {k: (json.dumps(v) if isinstance(v, (dict, list)) else v) for k, v in row.items()}
|
|
119
|
+
writer.writerow(row_clean)
|
|
120
|
+
print(f"Successfully exported {len(rows)} rows to CSV: {args.output}")
|
|
121
|
+
|
|
122
|
+
elif is_sqlite:
|
|
123
|
+
headers = list(rows[0].keys())
|
|
124
|
+
conn = sqlite3.connect(args.output)
|
|
125
|
+
cursor = conn.cursor()
|
|
126
|
+
|
|
127
|
+
# Dynamic table creation based on BigQuery results schema
|
|
128
|
+
# Maps common types, defaults everything else to TEXT
|
|
129
|
+
columns_def = []
|
|
130
|
+
for key, val in rows[0].items():
|
|
131
|
+
if isinstance(val, int):
|
|
132
|
+
col_type = "INTEGER"
|
|
133
|
+
elif isinstance(val, float):
|
|
134
|
+
col_type = "REAL"
|
|
135
|
+
else:
|
|
136
|
+
col_type = "TEXT"
|
|
137
|
+
columns_def.append(f"`{key}` {col_type}")
|
|
138
|
+
|
|
139
|
+
create_table_sql = f"CREATE TABLE IF NOT EXISTS `{args.table}` ({', '.join(columns_def)})"
|
|
140
|
+
cursor.execute(create_table_sql)
|
|
141
|
+
|
|
142
|
+
# Insert rows
|
|
143
|
+
insert_placeholders = ", ".join(["?"] * len(headers))
|
|
144
|
+
insert_sql = f"INSERT INTO `{args.table}` ({', '.join([f'`{h}`' for h in headers])}) VALUES ({insert_placeholders})"
|
|
145
|
+
|
|
146
|
+
insert_data = []
|
|
147
|
+
for row in rows:
|
|
148
|
+
row_data = []
|
|
149
|
+
for header in headers:
|
|
150
|
+
val = row[header]
|
|
151
|
+
if isinstance(val, (dict, list)):
|
|
152
|
+
row_data.append(json.dumps(val))
|
|
153
|
+
else:
|
|
154
|
+
row_data.append(val)
|
|
155
|
+
insert_data.append(tuple(row_data))
|
|
156
|
+
|
|
157
|
+
cursor.executemany(insert_sql, insert_data)
|
|
158
|
+
conn.commit()
|
|
159
|
+
conn.close()
|
|
160
|
+
print(f"Successfully imported {len(rows)} rows into SQLite table '{args.table}': {args.output}")
|
|
161
|
+
else:
|
|
162
|
+
print(f"Error: Unknown output format '{output_ext}'. Must be .json, .csv, or .db/.sqlite.")
|
|
163
|
+
sys.exit(1)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
if __name__ == "__main__":
|
|
167
|
+
main()
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
-- View: events_flat
|
|
2
|
+
-- Description: Flattens the nested time segments and interactions arrays in the latest logs
|
|
3
|
+
-- into a single clean event-stream format with one row per event.
|
|
4
|
+
-- Preserves the raw chronological time segments array for advanced engagement pattern analysis.
|
|
5
|
+
|
|
6
|
+
CREATE OR REPLACE VIEW `analytics.events_flat` AS
|
|
7
|
+
-- 1. Route visits (emitted when a route is closed/navigated away from)
|
|
8
|
+
SELECT
|
|
9
|
+
document_id,
|
|
10
|
+
__userId AS user_id,
|
|
11
|
+
__sessionId AS session_id,
|
|
12
|
+
__deviceId AS device_id,
|
|
13
|
+
route,
|
|
14
|
+
_createdAt,
|
|
15
|
+
_updatedAt,
|
|
16
|
+
'route_visit' AS event_type,
|
|
17
|
+
startTime AS timestamp,
|
|
18
|
+
CAST(NULL AS STRING) AS category,
|
|
19
|
+
CAST(NULL AS STRING) AS action,
|
|
20
|
+
CAST(NULL AS STRING) AS properties,
|
|
21
|
+
-- Preserves the chronological pattern of active/rest intervals (JSON array)
|
|
22
|
+
time_segments,
|
|
23
|
+
COALESCE(
|
|
24
|
+
(SELECT SUM(CAST(JSON_VALUE(t, '$.total') AS INT64))
|
|
25
|
+
FROM UNNEST(JSON_QUERY_ARRAY(time_segments)) t
|
|
26
|
+
WHERE JSON_VALUE(t, '$.type') = 'active'),
|
|
27
|
+
0
|
|
28
|
+
) AS total_active_ms,
|
|
29
|
+
COALESCE(
|
|
30
|
+
(SELECT SUM(CAST(JSON_VALUE(t, '$.total') AS INT64))
|
|
31
|
+
FROM UNNEST(JSON_QUERY_ARRAY(time_segments)) t
|
|
32
|
+
WHERE JSON_VALUE(t, '$.type') = 'rest'),
|
|
33
|
+
0
|
|
34
|
+
) AS total_rest_ms
|
|
35
|
+
FROM
|
|
36
|
+
`analytics.logs_latest`
|
|
37
|
+
|
|
38
|
+
UNION ALL
|
|
39
|
+
|
|
40
|
+
-- 2. Interactions (discrete click, focus, and submission events)
|
|
41
|
+
SELECT
|
|
42
|
+
document_id,
|
|
43
|
+
__userId AS user_id,
|
|
44
|
+
__sessionId AS session_id,
|
|
45
|
+
__deviceId AS device_id,
|
|
46
|
+
route,
|
|
47
|
+
_createdAt,
|
|
48
|
+
_updatedAt,
|
|
49
|
+
'interaction' AS event_type,
|
|
50
|
+
CAST(JSON_VALUE(ix, '$[0]') AS INT64) AS timestamp,
|
|
51
|
+
JSON_VALUE(ix, '$[1]') AS category,
|
|
52
|
+
JSON_VALUE(ix, '$[2]') AS action,
|
|
53
|
+
JSON_VALUE(ix, '$[3]') AS properties,
|
|
54
|
+
-- Null for interactions
|
|
55
|
+
CAST(NULL AS STRING) AS time_segments,
|
|
56
|
+
CAST(NULL AS INT64) AS total_active_ms,
|
|
57
|
+
CAST(NULL AS INT64) AS total_rest_ms
|
|
58
|
+
FROM
|
|
59
|
+
`analytics.logs_latest`,
|
|
60
|
+
UNNEST(JSON_QUERY_ARRAY(interactions)) ix;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
-- View: logs_latest
|
|
2
|
+
-- Description: Extracts the latest version of each log document from the raw changelog table,
|
|
3
|
+
-- deduplicating documents that have been updated (e.g., via sendBeacon writes).
|
|
4
|
+
|
|
5
|
+
CREATE OR REPLACE VIEW `analytics.logs_latest` AS
|
|
6
|
+
SELECT
|
|
7
|
+
document_id,
|
|
8
|
+
timestamp AS bq_sync_timestamp,
|
|
9
|
+
operation,
|
|
10
|
+
JSON_VALUE(data, '$.__userId') AS __userId,
|
|
11
|
+
JSON_VALUE(data, '$.__sessionId') AS __sessionId,
|
|
12
|
+
JSON_VALUE(data, '$.__deviceId') AS __deviceId,
|
|
13
|
+
JSON_VALUE(data, '$.route') AS route,
|
|
14
|
+
CAST(JSON_VALUE(data, '$.startTime') AS INT64) AS startTime,
|
|
15
|
+
JSON_QUERY(data, '$.time') AS time_segments,
|
|
16
|
+
JSON_QUERY(data, '$.interactions') AS interactions,
|
|
17
|
+
CAST(JSON_VALUE(data, '$._createdAt') AS INT64) AS _createdAt,
|
|
18
|
+
CAST(JSON_VALUE(data, '$._updatedAt') AS INT64) AS _updatedAt
|
|
19
|
+
FROM (
|
|
20
|
+
SELECT
|
|
21
|
+
document_id,
|
|
22
|
+
timestamp,
|
|
23
|
+
operation,
|
|
24
|
+
data,
|
|
25
|
+
ROW_NUMBER() OVER (
|
|
26
|
+
PARTITION BY document_id
|
|
27
|
+
ORDER BY timestamp DESC
|
|
28
|
+
) AS rn
|
|
29
|
+
FROM
|
|
30
|
+
`analytics.logs_raw_changelog`
|
|
31
|
+
)
|
|
32
|
+
WHERE
|
|
33
|
+
rn = 1
|
|
34
|
+
AND operation != 'DELETE';
|