@useairfoil/producer-google-analytics 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.txt +201 -0
- package/README.md +75 -0
- package/dist/index.d.mts +773 -0
- package/dist/index.mjs +3 -0
- package/dist/main.d.mts +1 -0
- package/dist/main.mjs +664 -0
- package/dist/main.mjs.map +1 -0
- package/dist/manifest.d.mts +45 -0
- package/dist/manifest.mjs +75 -0
- package/dist/manifest.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/dist/src-Cvr4Q9Xs.mjs +710 -0
- package/dist/src-Cvr4Q9Xs.mjs.map +1 -0
- package/package.json +46 -8
- package/src/client/auth.ts +133 -0
- package/src/client/client.ts +212 -0
- package/src/connector.ts +55 -0
- package/src/create-table.ts +10 -0
- package/src/demo/index.ts +44 -0
- package/src/demo/reports.ts +251 -0
- package/src/demo/shared.ts +98 -0
- package/src/index.ts +26 -0
- package/src/main.ts +28 -0
- package/src/manifest.ts +50 -0
- package/src/resources/reports/read.ts +104 -0
- package/src/resources/reports/resource.ts +31 -0
- package/src/resources/reports/sources.ts +169 -0
- package/src/resources/reports/spec.ts +15 -0
- package/src/resources/reports/specs.ts +173 -0
- package/src/sandbox.ts +37 -0
- package/src/schemas/api.ts +37 -0
- package/src/schemas/reports.ts +168 -0
- package/src/schemas/shared.ts +46 -0
- package/src/start.ts +49 -0
- package/src/tables.ts +25 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { ConnectorApp, Demo, RuntimeConfig } from "@useairfoil/connector-kit";
|
|
2
|
+
import { Command } from "effect/cli";
|
|
3
|
+
|
|
4
|
+
import {
|
|
5
|
+
DailyOverviewSchema,
|
|
6
|
+
DeviceSchema,
|
|
7
|
+
EcommerceItemSchema,
|
|
8
|
+
EventSchema,
|
|
9
|
+
LandingPageSchema,
|
|
10
|
+
LocationSchema,
|
|
11
|
+
PageSchema,
|
|
12
|
+
TrafficSourceSchema,
|
|
13
|
+
UserAcquisitionSchema,
|
|
14
|
+
} from "../schemas/reports";
|
|
15
|
+
import { dayRows } from "./reports";
|
|
16
|
+
|
|
17
|
+
export const demo = Demo.define({
|
|
18
|
+
name: "producer-google-analytics",
|
|
19
|
+
setup: { tables: [], rows: () => ({}) },
|
|
20
|
+
group: {
|
|
21
|
+
tables: [
|
|
22
|
+
Demo.table({ name: "daily_overview", rowSchema: DailyOverviewSchema, version: "version" }),
|
|
23
|
+
Demo.table({ name: "traffic_sources", rowSchema: TrafficSourceSchema, version: "version" }),
|
|
24
|
+
Demo.table({
|
|
25
|
+
name: "user_acquisition",
|
|
26
|
+
rowSchema: UserAcquisitionSchema,
|
|
27
|
+
version: "version",
|
|
28
|
+
}),
|
|
29
|
+
Demo.table({ name: "pages", rowSchema: PageSchema, version: "version" }),
|
|
30
|
+
Demo.table({ name: "landing_pages", rowSchema: LandingPageSchema, version: "version" }),
|
|
31
|
+
Demo.table({ name: "events", rowSchema: EventSchema, version: "version" }),
|
|
32
|
+
Demo.table({ name: "locations", rowSchema: LocationSchema, version: "version" }),
|
|
33
|
+
Demo.table({ name: "devices", rowSchema: DeviceSchema, version: "version" }),
|
|
34
|
+
Demo.table({ name: "ecommerce_items", rowSchema: EcommerceItemSchema, version: "version" }),
|
|
35
|
+
],
|
|
36
|
+
rows: dayRows,
|
|
37
|
+
// Two years of days, the same as the default GA_START_DATE.
|
|
38
|
+
count: 730,
|
|
39
|
+
},
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
export const demoCommand = ConnectorApp.makeDemoCommand(demo).pipe(
|
|
43
|
+
Command.provide(RuntimeConfig.layerHosted()),
|
|
44
|
+
);
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import type { Demo } from "@useairfoil/connector-kit";
|
|
2
|
+
|
|
3
|
+
import { DateTime } from "effect";
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
currency,
|
|
7
|
+
day,
|
|
8
|
+
devices,
|
|
9
|
+
host,
|
|
10
|
+
items,
|
|
11
|
+
locations,
|
|
12
|
+
money,
|
|
13
|
+
pages,
|
|
14
|
+
propertyId,
|
|
15
|
+
type Rows,
|
|
16
|
+
sources,
|
|
17
|
+
split,
|
|
18
|
+
} from "./shared";
|
|
19
|
+
|
|
20
|
+
/** One group is one day of reports, counting back from yesterday. */
|
|
21
|
+
export const dayRows = (index: number, { faker, now }: Demo.DemoContext): Rows => {
|
|
22
|
+
const date = DateTime.formatIsoDateUtc(
|
|
23
|
+
DateTime.fromDateUnsafe(new Date(now.getTime() - (index + 1) * day)),
|
|
24
|
+
);
|
|
25
|
+
|
|
26
|
+
const id = (...dimensions: ReadonlyArray<string>) =>
|
|
27
|
+
JSON.stringify([propertyId, date, ...dimensions]);
|
|
28
|
+
|
|
29
|
+
const base = { version: now, property_id: propertyId, date };
|
|
30
|
+
|
|
31
|
+
// Traffic grows over time and dips at the weekend.
|
|
32
|
+
const weekday = new Date(`${date}T00:00:00Z`).getUTCDay();
|
|
33
|
+
const weekend = weekday === 0 || weekday === 6 ? 0.75 : 1;
|
|
34
|
+
const growth = Math.max(0.4, 1 - index / 900);
|
|
35
|
+
|
|
36
|
+
const sessions = Math.round(
|
|
37
|
+
4200 * weekend * growth * faker.number.float({ min: 0.85, max: 1.15 }),
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
const engaged = Math.round(sessions * faker.number.float({ min: 0.55, max: 0.68 }));
|
|
41
|
+
const activeUsers = Math.round(sessions * 0.78);
|
|
42
|
+
const newUsers = Math.round(activeUsers * faker.number.float({ min: 0.42, max: 0.55 }));
|
|
43
|
+
const views = Math.round(sessions * faker.number.float({ min: 3.2, max: 4.4 }));
|
|
44
|
+
const purchases = Math.round(sessions * faker.number.float({ min: 0.018, max: 0.028 }));
|
|
45
|
+
const revenue = money(purchases * faker.number.float({ min: 58, max: 74 }));
|
|
46
|
+
const engagement = money(engaged * faker.number.float({ min: 95, max: 140 }));
|
|
47
|
+
|
|
48
|
+
const bySource = split(
|
|
49
|
+
sessions,
|
|
50
|
+
sources.map((source) => source[4]),
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
const byNewSource = split(
|
|
54
|
+
newUsers,
|
|
55
|
+
sources.map((source) => source[4]),
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
const purchasesBySource = split(
|
|
59
|
+
purchases,
|
|
60
|
+
sources.map((source) => source[4]),
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
const share = (part: number, whole: number) => (whole === 0 ? 0 : part / whole);
|
|
64
|
+
|
|
65
|
+
const eventCounts: ReadonlyArray<readonly [string, number, number]> = [
|
|
66
|
+
["page_view", views, activeUsers],
|
|
67
|
+
["session_start", sessions, activeUsers],
|
|
68
|
+
["first_visit", newUsers, newUsers],
|
|
69
|
+
["user_engagement", engaged * 3, Math.round(activeUsers * 0.7)],
|
|
70
|
+
["scroll", Math.round(views * 0.45), Math.round(activeUsers * 0.5)],
|
|
71
|
+
["view_item", Math.round(views * 0.3), Math.round(activeUsers * 0.4)],
|
|
72
|
+
["add_to_cart", Math.round(sessions * 0.08), Math.round(activeUsers * 0.07)],
|
|
73
|
+
["begin_checkout", Math.round(sessions * 0.04), Math.round(activeUsers * 0.035)],
|
|
74
|
+
["purchase", purchases, purchases],
|
|
75
|
+
];
|
|
76
|
+
|
|
77
|
+
// Cheaper items sell more units.
|
|
78
|
+
const itemUnits = split(
|
|
79
|
+
purchases + Math.round(purchases * 0.4),
|
|
80
|
+
items.map((item) => 10 - item[3] / 10),
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
const viewsByPage = split(
|
|
84
|
+
views,
|
|
85
|
+
pages.map((page) => page[2]),
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
const sessionsByLanding = split(sessions, [42, 16, 14, 10, 10, 8]);
|
|
89
|
+
|
|
90
|
+
const sessionsByLocation = split(
|
|
91
|
+
sessions,
|
|
92
|
+
locations.map((location) => location[3]),
|
|
93
|
+
);
|
|
94
|
+
|
|
95
|
+
const sessionsByDevice = split(
|
|
96
|
+
sessions,
|
|
97
|
+
devices.map((device) => device[4]),
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
return {
|
|
101
|
+
daily_overview: [
|
|
102
|
+
{
|
|
103
|
+
id: id(),
|
|
104
|
+
...base,
|
|
105
|
+
active_users: BigInt(activeUsers),
|
|
106
|
+
new_users: BigInt(newUsers),
|
|
107
|
+
total_users: BigInt(Math.round(activeUsers * 1.08)),
|
|
108
|
+
sessions: BigInt(sessions),
|
|
109
|
+
engaged_sessions: BigInt(engaged),
|
|
110
|
+
screen_page_views: BigInt(views),
|
|
111
|
+
event_count: BigInt(eventCounts.reduce((sum, [, count]) => sum + count, 0)),
|
|
112
|
+
key_events: purchases,
|
|
113
|
+
total_revenue: revenue,
|
|
114
|
+
user_engagement_duration: engagement,
|
|
115
|
+
currency_code: currency,
|
|
116
|
+
},
|
|
117
|
+
],
|
|
118
|
+
traffic_sources: sources.map(([source, medium, campaign, channel], s) => {
|
|
119
|
+
const part = bySource[s] ?? 0;
|
|
120
|
+
|
|
121
|
+
return {
|
|
122
|
+
id: id(source, medium, campaign, channel),
|
|
123
|
+
...base,
|
|
124
|
+
session_source: source,
|
|
125
|
+
session_medium: medium,
|
|
126
|
+
session_campaign_name: campaign,
|
|
127
|
+
session_default_channel_group: channel,
|
|
128
|
+
sessions: BigInt(part),
|
|
129
|
+
engaged_sessions: BigInt(Math.round(part * share(engaged, sessions))),
|
|
130
|
+
active_users: BigInt(Math.round(part * share(activeUsers, sessions))),
|
|
131
|
+
new_users: BigInt(Math.round(part * share(newUsers, sessions))),
|
|
132
|
+
screen_page_views: BigInt(Math.round(part * share(views, sessions))),
|
|
133
|
+
event_count: BigInt(Math.round(part * share(views, sessions) * 2.6)),
|
|
134
|
+
key_events: purchasesBySource[s] ?? 0,
|
|
135
|
+
total_revenue: money(revenue * share(purchasesBySource[s] ?? 0, purchases)),
|
|
136
|
+
user_engagement_duration: money(engagement * share(part, sessions)),
|
|
137
|
+
currency_code: currency,
|
|
138
|
+
};
|
|
139
|
+
}),
|
|
140
|
+
user_acquisition: sources.map(([source, medium, campaign, channel], s) => ({
|
|
141
|
+
id: id(source, medium, campaign, channel),
|
|
142
|
+
...base,
|
|
143
|
+
first_user_source: source,
|
|
144
|
+
first_user_medium: medium,
|
|
145
|
+
first_user_campaign_name: campaign,
|
|
146
|
+
first_user_default_channel_group: channel,
|
|
147
|
+
new_users: BigInt(byNewSource[s] ?? 0),
|
|
148
|
+
active_users: BigInt(Math.round((byNewSource[s] ?? 0) * 1.6)),
|
|
149
|
+
sessions: BigInt(Math.round((byNewSource[s] ?? 0) * 1.9)),
|
|
150
|
+
engaged_sessions: BigInt(Math.round((byNewSource[s] ?? 0) * 1.1)),
|
|
151
|
+
key_events: Math.round((purchasesBySource[s] ?? 0) * 0.4),
|
|
152
|
+
total_revenue: money(revenue * 0.4 * share(purchasesBySource[s] ?? 0, purchases)),
|
|
153
|
+
currency_code: currency,
|
|
154
|
+
})),
|
|
155
|
+
pages: pages.map(([path, title, weight], p) => {
|
|
156
|
+
const part = viewsByPage[p] ?? 0;
|
|
157
|
+
|
|
158
|
+
return {
|
|
159
|
+
id: id(host, path, title),
|
|
160
|
+
...base,
|
|
161
|
+
host_name: host,
|
|
162
|
+
page_path: path,
|
|
163
|
+
page_title: title,
|
|
164
|
+
screen_page_views: BigInt(part),
|
|
165
|
+
active_users: BigInt(Math.round(part * 0.62)),
|
|
166
|
+
event_count: BigInt(Math.round(part * 2.4)),
|
|
167
|
+
key_events: path === "/checkout" ? purchases : 0,
|
|
168
|
+
user_engagement_duration: money(engagement * (weight / 100)),
|
|
169
|
+
};
|
|
170
|
+
}),
|
|
171
|
+
landing_pages: pages.slice(0, 6).map(([path], l) => {
|
|
172
|
+
const part = sessionsByLanding[l] ?? 0;
|
|
173
|
+
|
|
174
|
+
return {
|
|
175
|
+
id: id(path),
|
|
176
|
+
...base,
|
|
177
|
+
landing_page: path,
|
|
178
|
+
sessions: BigInt(part),
|
|
179
|
+
engaged_sessions: BigInt(Math.round(part * share(engaged, sessions))),
|
|
180
|
+
active_users: BigInt(Math.round(part * 0.8)),
|
|
181
|
+
new_users: BigInt(Math.round(part * share(newUsers, sessions))),
|
|
182
|
+
key_events: Math.round(purchases * share(part, sessions)),
|
|
183
|
+
total_revenue: money(revenue * share(part, sessions)),
|
|
184
|
+
currency_code: currency,
|
|
185
|
+
};
|
|
186
|
+
}),
|
|
187
|
+
events: eventCounts.map(([name, count, users]) => ({
|
|
188
|
+
id: id(name),
|
|
189
|
+
...base,
|
|
190
|
+
event_name: name,
|
|
191
|
+
event_count: BigInt(count),
|
|
192
|
+
total_users: BigInt(users),
|
|
193
|
+
event_value: name === "purchase" ? revenue : 0,
|
|
194
|
+
key_events: name === "purchase" ? purchases : 0,
|
|
195
|
+
})),
|
|
196
|
+
locations: locations.map(([country, region, city], l) => {
|
|
197
|
+
const part = sessionsByLocation[l] ?? 0;
|
|
198
|
+
|
|
199
|
+
return {
|
|
200
|
+
id: id(country, region, city),
|
|
201
|
+
...base,
|
|
202
|
+
country,
|
|
203
|
+
region,
|
|
204
|
+
city,
|
|
205
|
+
active_users: BigInt(Math.round(part * 0.78)),
|
|
206
|
+
new_users: BigInt(Math.round(part * share(newUsers, sessions))),
|
|
207
|
+
sessions: BigInt(part),
|
|
208
|
+
engaged_sessions: BigInt(Math.round(part * share(engaged, sessions))),
|
|
209
|
+
key_events: Math.round(purchases * share(part, sessions)),
|
|
210
|
+
total_revenue: money(revenue * share(part, sessions)),
|
|
211
|
+
currency_code: currency,
|
|
212
|
+
};
|
|
213
|
+
}),
|
|
214
|
+
devices: devices.map(([category, platform, os, browser], d) => {
|
|
215
|
+
const part = sessionsByDevice[d] ?? 0;
|
|
216
|
+
|
|
217
|
+
return {
|
|
218
|
+
id: id(category, platform, os, browser),
|
|
219
|
+
...base,
|
|
220
|
+
device_category: category,
|
|
221
|
+
platform,
|
|
222
|
+
operating_system: os,
|
|
223
|
+
browser,
|
|
224
|
+
active_users: BigInt(Math.round(part * 0.78)),
|
|
225
|
+
new_users: BigInt(Math.round(part * share(newUsers, sessions))),
|
|
226
|
+
sessions: BigInt(part),
|
|
227
|
+
engaged_sessions: BigInt(Math.round(part * share(engaged, sessions))),
|
|
228
|
+
screen_page_views: BigInt(Math.round(part * share(views, sessions))),
|
|
229
|
+
key_events: Math.round(purchases * share(part, sessions)),
|
|
230
|
+
};
|
|
231
|
+
}),
|
|
232
|
+
ecommerce_items: items.map(([itemId, name, category, price], i) => {
|
|
233
|
+
const bought = itemUnits[i] ?? 0;
|
|
234
|
+
const added = Math.round(bought * faker.number.float({ min: 3, max: 5 }));
|
|
235
|
+
|
|
236
|
+
return {
|
|
237
|
+
id: id(itemId, name, "Acme", category),
|
|
238
|
+
...base,
|
|
239
|
+
item_id: itemId,
|
|
240
|
+
item_name: name,
|
|
241
|
+
item_brand: "Acme",
|
|
242
|
+
item_category: category,
|
|
243
|
+
items_viewed: BigInt(Math.round(added * faker.number.float({ min: 6, max: 9 }))),
|
|
244
|
+
items_added_to_cart: BigInt(added),
|
|
245
|
+
items_purchased: BigInt(bought),
|
|
246
|
+
item_revenue: money(bought * price),
|
|
247
|
+
currency_code: currency,
|
|
248
|
+
};
|
|
249
|
+
}),
|
|
250
|
+
};
|
|
251
|
+
};
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { Duration } from "effect";
|
|
2
|
+
|
|
3
|
+
import type {
|
|
4
|
+
DailyOverview,
|
|
5
|
+
Device,
|
|
6
|
+
EcommerceItem,
|
|
7
|
+
Event,
|
|
8
|
+
LandingPage,
|
|
9
|
+
Location,
|
|
10
|
+
Page,
|
|
11
|
+
TrafficSource,
|
|
12
|
+
UserAcquisition,
|
|
13
|
+
} from "../schemas/reports";
|
|
14
|
+
|
|
15
|
+
export const day = Duration.toMillis(Duration.days(1));
|
|
16
|
+
export const propertyId = "123456789";
|
|
17
|
+
export const host = "shop.acme.example.com";
|
|
18
|
+
export const currency = "USD";
|
|
19
|
+
|
|
20
|
+
// [source, medium, campaign, channel group, share of sessions]
|
|
21
|
+
export const sources = [
|
|
22
|
+
["google", "organic", "(organic)", "Organic Search", 34],
|
|
23
|
+
["(direct)", "(none)", "(direct)", "Direct", 22],
|
|
24
|
+
["google", "cpc", "brand_search", "Paid Search", 14],
|
|
25
|
+
["facebook", "paid_social", "fall_sale", "Paid Social", 9],
|
|
26
|
+
["newsletter", "email", "weekly_digest", "Email", 8],
|
|
27
|
+
["bing", "organic", "(organic)", "Organic Search", 5],
|
|
28
|
+
["instagram.com", "referral", "(referral)", "Organic Social", 5],
|
|
29
|
+
["partner.example.com", "referral", "(referral)", "Referral", 3],
|
|
30
|
+
] as const;
|
|
31
|
+
|
|
32
|
+
export const pages = [
|
|
33
|
+
["/", "Acme - Everyday goods", 30],
|
|
34
|
+
["/collections/new", "New arrivals - Acme", 14],
|
|
35
|
+
["/collections/sale", "Sale - Acme", 12],
|
|
36
|
+
["/products/classic-tee", "Classic Tee - Acme", 9],
|
|
37
|
+
["/products/everyday-hoodie", "Everyday Hoodie - Acme", 8],
|
|
38
|
+
["/products/water-bottle", "Insulated Water Bottle - Acme", 6],
|
|
39
|
+
["/cart", "Your cart - Acme", 8],
|
|
40
|
+
["/checkout", "Checkout - Acme", 5],
|
|
41
|
+
["/blog/how-we-source-cotton", "How we source our cotton - Acme", 4],
|
|
42
|
+
["/pages/shipping", "Shipping - Acme", 4],
|
|
43
|
+
] as const;
|
|
44
|
+
|
|
45
|
+
export const locations = [
|
|
46
|
+
["United States", "California", "Los Angeles", 14],
|
|
47
|
+
["United States", "New York", "New York", 12],
|
|
48
|
+
["United States", "Texas", "Austin", 7],
|
|
49
|
+
["United States", "Illinois", "Chicago", 6],
|
|
50
|
+
["United Kingdom", "England", "London", 9],
|
|
51
|
+
["Canada", "Ontario", "Toronto", 6],
|
|
52
|
+
["Germany", "Berlin", "Berlin", 5],
|
|
53
|
+
["Australia", "New South Wales", "Sydney", 4],
|
|
54
|
+
["United States", "(not set)", "(not set)", 3],
|
|
55
|
+
] as const;
|
|
56
|
+
|
|
57
|
+
export const devices = [
|
|
58
|
+
["mobile", "web", "iOS", "Safari", 30],
|
|
59
|
+
["mobile", "web", "Android", "Chrome", 22],
|
|
60
|
+
["desktop", "web", "Windows", "Chrome", 18],
|
|
61
|
+
["desktop", "web", "Macintosh", "Chrome", 12],
|
|
62
|
+
["desktop", "web", "Macintosh", "Safari", 10],
|
|
63
|
+
["tablet", "web", "iOS", "Safari", 5],
|
|
64
|
+
["desktop", "web", "Windows", "Edge", 3],
|
|
65
|
+
] as const;
|
|
66
|
+
|
|
67
|
+
export const items = [
|
|
68
|
+
["SKU-TEE", "Classic Cotton Tee", "Apparel", 28],
|
|
69
|
+
["SKU-HOODIE", "Everyday Hoodie", "Apparel", 64],
|
|
70
|
+
["SKU-SHIRT", "Linen Button Shirt", "Apparel", 72],
|
|
71
|
+
["SKU-BEANIE", "Wool Beanie", "Accessories", 26],
|
|
72
|
+
["SKU-TOTE", "Canvas Tote Bag", "Accessories", 22],
|
|
73
|
+
["SKU-MUG", "Stoneware Mug", "Home", 18],
|
|
74
|
+
["SKU-BOTTLE", "Insulated Water Bottle", "Home", 32],
|
|
75
|
+
] as const;
|
|
76
|
+
|
|
77
|
+
/** Splits a total by weights into whole numbers that add up to it. */
|
|
78
|
+
export const split = (total: number, weights: ReadonlyArray<number>) => {
|
|
79
|
+
const sum = weights.reduce((a, b) => a + b, 0);
|
|
80
|
+
const parts = weights.map((weight) => Math.floor((total * weight) / sum));
|
|
81
|
+
parts[0] = (parts[0] ?? 0) + total - parts.reduce((a, b) => a + b, 0);
|
|
82
|
+
|
|
83
|
+
return parts;
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
export const money = (value: number) => Math.round(value * 100) / 100;
|
|
87
|
+
|
|
88
|
+
export type Rows = {
|
|
89
|
+
readonly daily_overview: Array<DailyOverview>;
|
|
90
|
+
readonly traffic_sources: Array<TrafficSource>;
|
|
91
|
+
readonly user_acquisition: Array<UserAcquisition>;
|
|
92
|
+
readonly pages: Array<Page>;
|
|
93
|
+
readonly landing_pages: Array<LandingPage>;
|
|
94
|
+
readonly events: Array<Event>;
|
|
95
|
+
readonly locations: Array<Location>;
|
|
96
|
+
readonly devices: Array<Device>;
|
|
97
|
+
readonly ecommerce_items: Array<EcommerceItem>;
|
|
98
|
+
};
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export * as GoogleAnalyticsAuth from "./client/auth";
|
|
2
|
+
export * as GoogleAnalyticsClient from "./client/client";
|
|
3
|
+
export * as GoogleAnalyticsConnector from "./connector";
|
|
4
|
+
export { manifest } from "./manifest";
|
|
5
|
+
export type {
|
|
6
|
+
DailyOverview,
|
|
7
|
+
Device,
|
|
8
|
+
EcommerceItem,
|
|
9
|
+
Event,
|
|
10
|
+
LandingPage,
|
|
11
|
+
Location,
|
|
12
|
+
Page,
|
|
13
|
+
TrafficSource,
|
|
14
|
+
UserAcquisition,
|
|
15
|
+
} from "./schemas/reports";
|
|
16
|
+
export {
|
|
17
|
+
DailyOverviewSchema,
|
|
18
|
+
DeviceSchema,
|
|
19
|
+
EcommerceItemSchema,
|
|
20
|
+
EventSchema,
|
|
21
|
+
LandingPageSchema,
|
|
22
|
+
LocationSchema,
|
|
23
|
+
PageSchema,
|
|
24
|
+
TrafficSourceSchema,
|
|
25
|
+
UserAcquisitionSchema,
|
|
26
|
+
} from "./schemas/reports";
|
package/src/main.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { NodeRuntime, NodeServices } from "@effect/platform-node";
|
|
4
|
+
import { Effect, Layer } from "effect";
|
|
5
|
+
import { Command } from "effect/cli";
|
|
6
|
+
import { FetchHttpClient } from "effect/http";
|
|
7
|
+
|
|
8
|
+
import packageJson from "../package.json";
|
|
9
|
+
import { createTableCommand, setupCommand } from "./create-table";
|
|
10
|
+
import { demoCommand } from "./demo";
|
|
11
|
+
import { sandboxCommand } from "./sandbox";
|
|
12
|
+
import { startCommand } from "./start";
|
|
13
|
+
|
|
14
|
+
const BootstrapLayer = Layer.mergeAll(FetchHttpClient.layer, NodeServices.layer);
|
|
15
|
+
|
|
16
|
+
const program = Command.make("producer-google-analytics", {}, () => Effect.void).pipe(
|
|
17
|
+
Command.withSubcommands([
|
|
18
|
+
setupCommand,
|
|
19
|
+
startCommand,
|
|
20
|
+
sandboxCommand,
|
|
21
|
+
createTableCommand,
|
|
22
|
+
demoCommand,
|
|
23
|
+
]),
|
|
24
|
+
);
|
|
25
|
+
|
|
26
|
+
const cli = Command.run(program, { version: packageJson.version });
|
|
27
|
+
|
|
28
|
+
NodeRuntime.runMain(cli.pipe(Effect.provide(BootstrapLayer), Effect.scoped));
|
package/src/manifest.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import * as Manifest from "@useairfoil/connector-kit/manifest";
|
|
2
|
+
|
|
3
|
+
export const GoogleAnalyticsConfigDef = Manifest.defineConfig({
|
|
4
|
+
propertyId: Manifest.string({
|
|
5
|
+
runtimeKey: "GA_PROPERTY_ID",
|
|
6
|
+
description: "Numeric ID of the GA4 property, from Admin > Property details.",
|
|
7
|
+
}),
|
|
8
|
+
clientEmail: Manifest.string({
|
|
9
|
+
runtimeKey: "GA_CLIENT_EMAIL",
|
|
10
|
+
description: "`client_email` from the service account's JSON key.",
|
|
11
|
+
}),
|
|
12
|
+
privateKey: Manifest.secret({
|
|
13
|
+
runtimeKey: "GA_PRIVATE_KEY",
|
|
14
|
+
description: "`private_key` from the service account's JSON key.",
|
|
15
|
+
}),
|
|
16
|
+
startDate: Manifest.optional(
|
|
17
|
+
Manifest.string({
|
|
18
|
+
runtimeKey: "GA_START_DATE",
|
|
19
|
+
description: "First day to backfill, as YYYY-MM-DD. Defaults to 2 years back.",
|
|
20
|
+
}),
|
|
21
|
+
),
|
|
22
|
+
lookbackDays: Manifest.number({
|
|
23
|
+
runtimeKey: "GA_LOOKBACK_DAYS",
|
|
24
|
+
description: "Days read again on each run, since Google keeps updating recent data.",
|
|
25
|
+
integer: true,
|
|
26
|
+
minimum: 1,
|
|
27
|
+
default: 14,
|
|
28
|
+
}),
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
export type GoogleAnalyticsConfig = Manifest.ConfigValuesOf<typeof GoogleAnalyticsConfigDef>;
|
|
32
|
+
|
|
33
|
+
const capabilities = ["backfill", "changes"] as const;
|
|
34
|
+
|
|
35
|
+
export const manifest = Manifest.define({
|
|
36
|
+
name: "producer-google-analytics",
|
|
37
|
+
title: "Google Analytics",
|
|
38
|
+
config: GoogleAnalyticsConfigDef.spec,
|
|
39
|
+
resources: [
|
|
40
|
+
{ name: "daily_overview", capabilities },
|
|
41
|
+
{ name: "traffic_sources", capabilities },
|
|
42
|
+
{ name: "user_acquisition", capabilities },
|
|
43
|
+
{ name: "pages", capabilities },
|
|
44
|
+
{ name: "landing_pages", capabilities },
|
|
45
|
+
{ name: "events", capabilities },
|
|
46
|
+
{ name: "locations", capabilities },
|
|
47
|
+
{ name: "devices", capabilities },
|
|
48
|
+
{ name: "ecommerce_items", capabilities },
|
|
49
|
+
],
|
|
50
|
+
});
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { ConnectorError } from "@useairfoil/connector-kit";
|
|
2
|
+
import { DateTime, Effect, Option, Schema } from "effect";
|
|
3
|
+
|
|
4
|
+
import type { GoogleAnalyticsClientService } from "../../client/client";
|
|
5
|
+
import type { RunReportResponse } from "../../schemas/api";
|
|
6
|
+
import type { ReportSpec } from "./spec";
|
|
7
|
+
|
|
8
|
+
import { isoDay } from "../../schemas/shared";
|
|
9
|
+
|
|
10
|
+
// The API returns at most 250,000 rows a request. Smaller pages keep memory low.
|
|
11
|
+
const pageSize = 100_000;
|
|
12
|
+
|
|
13
|
+
export type ReportPage<Row> = {
|
|
14
|
+
readonly rows: ReadonlyArray<Row>;
|
|
15
|
+
/** Offset of the next page of the same day. */
|
|
16
|
+
readonly next: Option.Option<number>;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
type ReportRow = NonNullable<RunReportResponse["rows"]>[number];
|
|
20
|
+
|
|
21
|
+
/** `[column, apiName]` pairs, in request order. */
|
|
22
|
+
const pairs = (columns: Readonly<Record<string, string | undefined>>) =>
|
|
23
|
+
Object.entries(columns).flatMap(([column, name]) =>
|
|
24
|
+
name === undefined ? [] : [[column, name] as const],
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
const valuesByName = (
|
|
28
|
+
headers: ReadonlyArray<{ readonly name: string }>,
|
|
29
|
+
values: ReadonlyArray<{ readonly value: string }>,
|
|
30
|
+
) => new Map(headers.map((header, index) => [header.name, values[index]?.value]));
|
|
31
|
+
|
|
32
|
+
/** Logs the response flags that mean some data is hidden or estimated. */
|
|
33
|
+
const warnDataLoss = (name: string, day: string, metadata: RunReportResponse["metadata"]) => {
|
|
34
|
+
const flags = {
|
|
35
|
+
otherRow: metadata.dataLossFromOtherRow === true,
|
|
36
|
+
thresholded: metadata.subjectToThresholding === true,
|
|
37
|
+
sampled: (metadata.samplingMetadatas?.length ?? 0) > 0,
|
|
38
|
+
truncated: (metadata.dataTruncationReasons?.length ?? 0) > 0,
|
|
39
|
+
};
|
|
40
|
+
return Object.values(flags).some(Boolean)
|
|
41
|
+
? Effect.logWarning("Google Analytics report is incomplete").pipe(
|
|
42
|
+
Effect.annotateLogs({ report: name, day, ...flags }),
|
|
43
|
+
)
|
|
44
|
+
: Effect.void;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export const readDay = <Row extends object>(
|
|
48
|
+
client: GoogleAnalyticsClientService,
|
|
49
|
+
propertyId: string,
|
|
50
|
+
spec: ReportSpec<Row>,
|
|
51
|
+
options: { readonly day: string; readonly offset: number; readonly limit?: number },
|
|
52
|
+
): Effect.Effect<ReportPage<Row>, ConnectorError> =>
|
|
53
|
+
Effect.gen(function* () {
|
|
54
|
+
const dimensions = pairs(spec.dimensions);
|
|
55
|
+
const metrics = pairs(spec.metrics);
|
|
56
|
+
const columns = [...dimensions, ...metrics];
|
|
57
|
+
const response = yield* client.runReport(spec.name, {
|
|
58
|
+
dimensions: ["date", ...dimensions.map(([, name]) => name)],
|
|
59
|
+
metrics: metrics.map(([, name]) => name),
|
|
60
|
+
startDate: options.day,
|
|
61
|
+
endDate: options.day,
|
|
62
|
+
limit: options.limit ?? pageSize,
|
|
63
|
+
offset: options.offset,
|
|
64
|
+
});
|
|
65
|
+
yield* warnDataLoss(spec.name, options.day, response.metadata);
|
|
66
|
+
const version = yield* DateTime.nowAsDate;
|
|
67
|
+
const decodeRow = Schema.decodeUnknownEffect(spec.rowSchema);
|
|
68
|
+
|
|
69
|
+
const toRow = (row: ReportRow) => {
|
|
70
|
+
const values = new Map([
|
|
71
|
+
...valuesByName(response.dimensionHeaders, row.dimensionValues),
|
|
72
|
+
...valuesByName(response.metricHeaders, row.metricValues),
|
|
73
|
+
]);
|
|
74
|
+
const date = values.get("date") ?? "";
|
|
75
|
+
return decodeRow({
|
|
76
|
+
id: JSON.stringify([
|
|
77
|
+
propertyId,
|
|
78
|
+
isoDay(date),
|
|
79
|
+
...dimensions.map(([, name]) => values.get(name)),
|
|
80
|
+
]),
|
|
81
|
+
version,
|
|
82
|
+
property_id: propertyId,
|
|
83
|
+
date,
|
|
84
|
+
// Tables without revenue leave it out.
|
|
85
|
+
currency_code: response.metadata.currencyCode,
|
|
86
|
+
...Object.fromEntries(columns.map(([column, name]) => [column, values.get(name)])),
|
|
87
|
+
}).pipe(
|
|
88
|
+
Effect.mapError(
|
|
89
|
+
(cause) =>
|
|
90
|
+
new ConnectorError({
|
|
91
|
+
message: `Unexpected Google Analytics row in ${spec.name} for ${options.day}`,
|
|
92
|
+
cause,
|
|
93
|
+
}),
|
|
94
|
+
),
|
|
95
|
+
);
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const rows = yield* Effect.forEach(response.rows ?? [], toRow);
|
|
99
|
+
const read = options.offset + rows.length;
|
|
100
|
+
return {
|
|
101
|
+
rows,
|
|
102
|
+
next: rows.length > 0 && read < (response.rowCount ?? 0) ? Option.some(read) : Option.none(),
|
|
103
|
+
};
|
|
104
|
+
});
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { ConnectorError, Identified } from "@useairfoil/connector-kit";
|
|
2
|
+
import type { Effect, Schema } from "effect";
|
|
3
|
+
|
|
4
|
+
import type { GoogleAnalyticsClientService } from "../../client/client";
|
|
5
|
+
import type { GoogleAnalyticsConfig } from "../../manifest";
|
|
6
|
+
import type { ReportSpec } from "./spec";
|
|
7
|
+
|
|
8
|
+
import { reportBackfill, reportChanges, reportCheck } from "./sources";
|
|
9
|
+
|
|
10
|
+
/** What `Resource.entity` takes for a report. */
|
|
11
|
+
export type ReportEntity<Row extends object> = {
|
|
12
|
+
readonly name: string;
|
|
13
|
+
readonly rowSchema: Schema.Decoder<Row> & Identified<"id">;
|
|
14
|
+
readonly version: "version";
|
|
15
|
+
readonly check: Effect.Effect<void, ConnectorError>;
|
|
16
|
+
readonly backfill: ReturnType<typeof reportBackfill<Row>>;
|
|
17
|
+
readonly changes: ReturnType<typeof reportChanges<Row>>;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
export const reportEntity = <Row extends object>(
|
|
21
|
+
client: GoogleAnalyticsClientService,
|
|
22
|
+
config: GoogleAnalyticsConfig,
|
|
23
|
+
spec: ReportSpec<Row>,
|
|
24
|
+
): ReportEntity<Row> => ({
|
|
25
|
+
name: spec.name,
|
|
26
|
+
rowSchema: spec.rowSchema,
|
|
27
|
+
version: "version",
|
|
28
|
+
check: reportCheck(client, config, spec),
|
|
29
|
+
backfill: reportBackfill(client, config, spec),
|
|
30
|
+
changes: reportChanges(client, config, spec),
|
|
31
|
+
});
|