@speedkit/cli 4.25.2 → 4.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/README.md +192 -19
  3. package/dist/commands/auto-prewarm.d.ts +8 -1
  4. package/dist/commands/auto-prewarm.js +18 -7
  5. package/dist/commands/generate-pop-config.d.ts +1 -0
  6. package/dist/commands/generate-pop-config.js +9 -3
  7. package/dist/commands/prewarm.js +3 -1
  8. package/dist/commands/query/parameter.d.ts +26 -0
  9. package/dist/commands/query/parameter.js +27 -0
  10. package/dist/commands/query/prewarm.d.ts +31 -0
  11. package/dist/commands/query/prewarm.js +31 -0
  12. package/dist/commands/revalidate.js +8 -6
  13. package/dist/helpers/clipboard.d.ts +19 -0
  14. package/dist/helpers/clipboard.js +70 -0
  15. package/dist/helpers/evaluate-speed-kit-config.d.ts +8 -18
  16. package/dist/helpers/evaluate-speed-kit-config.js +8 -6
  17. package/dist/helpers/evaluate-speed-kit-config.spec.d.ts +1 -0
  18. package/dist/helpers/evaluate-speed-kit-config.spec.js +78 -0
  19. package/dist/models/cli-parameters.d.ts +73 -0
  20. package/dist/models/cli-parameters.js +151 -0
  21. package/dist/models/cli-parameters.spec.d.ts +1 -0
  22. package/dist/models/cli-parameters.spec.js +47 -0
  23. package/dist/services/athena/athena-service.d.ts +2 -0
  24. package/dist/services/athena/athena-service.js +15 -4
  25. package/dist/services/athena/athena-service.spec.d.ts +1 -0
  26. package/dist/services/athena/athena-service.spec.js +74 -0
  27. package/dist/services/deploy/handler/install-resource-handler.js +3 -4
  28. package/dist/services/onboarding/onboarding-service-factory.js +2 -2
  29. package/dist/services/pop-config/pop-config-service.d.ts +8 -1
  30. package/dist/services/pop-config/pop-config-service.js +16 -28
  31. package/dist/services/prewarm/assets/asset-api-client.d.ts +1 -1
  32. package/dist/services/prewarm/assets/asset-api-client.js +5 -4
  33. package/dist/services/prewarm/auto-pre-warm-factory.d.ts +12 -2
  34. package/dist/services/prewarm/auto-pre-warm-factory.js +20 -6
  35. package/dist/services/prewarm/csv-reader.d.ts +21 -2
  36. package/dist/services/prewarm/csv-reader.js +71 -9
  37. package/dist/services/prewarm/csv-reader.spec.d.ts +1 -0
  38. package/dist/services/prewarm/csv-reader.spec.js +75 -0
  39. package/dist/services/prewarm/index.d.ts +1 -0
  40. package/dist/services/prewarm/index.js +1 -0
  41. package/dist/services/prewarm/pre-warm-factory.d.ts +0 -2
  42. package/dist/services/prewarm/pre-warm-factory.js +8 -11
  43. package/dist/services/prewarm/pre-warm-model.d.ts +14 -1
  44. package/dist/services/prewarm/pre-warm-model.js +0 -1
  45. package/dist/services/prewarm/pre-warm-service.d.ts +9 -4
  46. package/dist/services/prewarm/pre-warm-service.js +18 -14
  47. package/dist/services/prewarm/prewarm-targets.d.ts +12 -0
  48. package/dist/services/prewarm/prewarm-targets.js +18 -0
  49. package/dist/services/prewarm/prewarm-targets.spec.d.ts +1 -0
  50. package/dist/services/prewarm/prewarm-targets.spec.js +29 -0
  51. package/dist/services/query-builder/error/match-matches-nothing-error.d.ts +11 -0
  52. package/dist/services/query-builder/error/match-matches-nothing-error.js +17 -0
  53. package/dist/services/query-builder/queries/device.d.ts +21 -0
  54. package/dist/services/query-builder/queries/device.js +26 -0
  55. package/dist/services/query-builder/queries/index.d.ts +6 -0
  56. package/dist/services/query-builder/queries/index.js +6 -0
  57. package/dist/services/query-builder/queries/page-filter.d.ts +39 -0
  58. package/dist/services/query-builder/queries/page-filter.js +135 -0
  59. package/dist/services/query-builder/queries/page-source.d.ts +30 -0
  60. package/dist/services/query-builder/queries/page-source.js +79 -0
  61. package/dist/services/query-builder/queries/parameter-query.d.ts +14 -0
  62. package/dist/services/query-builder/queries/parameter-query.js +142 -0
  63. package/dist/services/query-builder/queries/pops-query.d.ts +27 -0
  64. package/dist/services/query-builder/queries/pops-query.js +65 -0
  65. package/dist/services/query-builder/queries/prewarm-query.d.ts +18 -0
  66. package/dist/services/query-builder/queries/prewarm-query.js +227 -0
  67. package/dist/services/query-builder/queries/query-model.d.ts +97 -0
  68. package/dist/services/query-builder/queries/query-model.js +19 -0
  69. package/dist/services/query-builder/queries/spec/golden-configs.d.ts +18 -0
  70. package/dist/services/query-builder/queries/spec/golden-configs.js +116 -0
  71. package/dist/services/query-builder/queries/spec/golden.spec.d.ts +1 -0
  72. package/dist/services/query-builder/queries/spec/golden.spec.js +161 -0
  73. package/dist/services/query-builder/queries/spec/pops-query.spec.d.ts +1 -0
  74. package/dist/services/query-builder/queries/spec/pops-query.spec.js +58 -0
  75. package/dist/services/query-builder/queries/spec/queries.spec.d.ts +1 -0
  76. package/dist/services/query-builder/queries/spec/queries.spec.js +312 -0
  77. package/dist/services/query-builder/queries/spec/reported-bugs.spec.d.ts +1 -0
  78. package/dist/services/query-builder/queries/spec/reported-bugs.spec.js +86 -0
  79. package/dist/services/query-builder/query-builder-factory.d.ts +2 -2
  80. package/dist/services/query-builder/query-builder-factory.js +35 -7
  81. package/dist/services/query-builder/query-builder-model.d.ts +31 -19
  82. package/dist/services/query-builder/query-builder-model.js +5 -5
  83. package/dist/services/query-builder/query-builder-service.d.ts +26 -5
  84. package/dist/services/query-builder/query-builder-service.js +157 -44
  85. package/dist/services/query-builder/query-command.d.ts +26 -0
  86. package/dist/services/query-builder/query-command.js +35 -0
  87. package/dist/services/query-builder/rules/index.d.ts +8 -0
  88. package/dist/services/query-builder/rules/index.js +8 -0
  89. package/dist/services/query-builder/rules/match-pattern.d.ts +51 -0
  90. package/dist/services/query-builder/rules/match-pattern.js +141 -0
  91. package/dist/services/query-builder/rules/pattern-language.d.ts +59 -0
  92. package/dist/services/query-builder/rules/pattern-language.js +159 -0
  93. package/dist/services/query-builder/rules/rule-model.d.ts +107 -0
  94. package/dist/services/query-builder/rules/rule-model.js +54 -0
  95. package/dist/services/query-builder/rules/rule-parser.d.ts +33 -0
  96. package/dist/services/query-builder/rules/rule-parser.js +189 -0
  97. package/dist/services/query-builder/rules/rule-reducer.d.ts +71 -0
  98. package/dist/services/query-builder/rules/rule-reducer.js +383 -0
  99. package/dist/services/query-builder/rules/rule-relevance.d.ts +43 -0
  100. package/dist/services/query-builder/rules/rule-relevance.js +69 -0
  101. package/dist/services/query-builder/rules/rule-to-sql.d.ts +51 -0
  102. package/dist/services/query-builder/rules/rule-to-sql.js +75 -0
  103. package/dist/services/query-builder/rules/spec/pattern-language.spec.d.ts +1 -0
  104. package/dist/services/query-builder/rules/spec/pattern-language.spec.js +90 -0
  105. package/dist/services/query-builder/rules/spec/rule-parser.spec.d.ts +1 -0
  106. package/dist/services/query-builder/rules/spec/rule-parser.spec.js +150 -0
  107. package/dist/services/query-builder/rules/spec/rule-reducer.spec.d.ts +1 -0
  108. package/dist/services/query-builder/rules/spec/rule-reducer.spec.js +169 -0
  109. package/dist/services/query-builder/rules/spec/rule-relevance.spec.d.ts +1 -0
  110. package/dist/services/query-builder/rules/spec/rule-relevance.spec.js +88 -0
  111. package/dist/services/query-builder/rules/spec/rule-to-sql.spec.d.ts +1 -0
  112. package/dist/services/query-builder/rules/spec/rule-to-sql.spec.js +110 -0
  113. package/dist/services/query-builder/rules/spec/runtime-semantics.spec.d.ts +1 -0
  114. package/dist/services/query-builder/rules/spec/runtime-semantics.spec.js +55 -0
  115. package/dist/services/query-builder/rules/spec/strip-parameters.spec.d.ts +1 -0
  116. package/dist/services/query-builder/rules/spec/strip-parameters.spec.js +85 -0
  117. package/dist/services/query-builder/rules/strip-parameters.d.ts +39 -0
  118. package/dist/services/query-builder/rules/strip-parameters.js +197 -0
  119. package/dist/services/query-builder/sql/expression.d.ts +73 -0
  120. package/dist/services/query-builder/sql/expression.js +133 -0
  121. package/dist/services/query-builder/sql/format.d.ts +14 -0
  122. package/dist/services/query-builder/sql/format.js +48 -0
  123. package/dist/services/query-builder/sql/index.d.ts +4 -0
  124. package/dist/services/query-builder/sql/index.js +4 -0
  125. package/dist/services/query-builder/sql/predicate.d.ts +66 -0
  126. package/dist/services/query-builder/sql/predicate.js +124 -0
  127. package/dist/services/query-builder/sql/regex.d.ts +28 -0
  128. package/dist/services/query-builder/sql/regex.js +81 -0
  129. package/dist/services/query-builder/sql/spec/expression.spec.d.ts +1 -0
  130. package/dist/services/query-builder/sql/spec/expression.spec.js +75 -0
  131. package/dist/services/query-builder/sql/spec/format.spec.d.ts +1 -0
  132. package/dist/services/query-builder/sql/spec/format.spec.js +37 -0
  133. package/dist/services/query-builder/sql/spec/predicate.spec.d.ts +1 -0
  134. package/dist/services/query-builder/sql/spec/predicate.spec.js +78 -0
  135. package/dist/services/query-builder/sql/spec/regex.spec.d.ts +1 -0
  136. package/dist/services/query-builder/sql/spec/regex.spec.js +62 -0
  137. package/dist/services/revalidate/revalidate-model.d.ts +9 -3
  138. package/dist/services/revalidate/revalidate-model.js +12 -3
  139. package/oclif.manifest.json +354 -104
  140. package/package.json +11 -2
  141. package/dist/commands/build-parameter-query.d.ts +0 -14
  142. package/dist/commands/build-parameter-query.js +0 -25
  143. package/dist/commands/build-prewarm-query.d.ts +0 -16
  144. package/dist/commands/build-prewarm-query.js +0 -37
  145. package/dist/helpers/build-query-helper.d.ts +0 -32
  146. package/dist/helpers/build-query-helper.js +0 -223
  147. package/dist/helpers/get-parsed-config.d.ts +0 -15
  148. package/dist/helpers/get-parsed-config.js +0 -62
  149. package/dist/services/onboarding/dashboard/parameter-query-builder.d.ts +0 -8
  150. package/dist/services/onboarding/dashboard/parameter-query-builder.js +0 -63
@@ -0,0 +1,142 @@
1
+ import { and, call, column, formatPredicate, greaterThan, literal, raw, } from "../sql/index.js";
2
+ import { notStrippedParameter, prepareStripParameters, } from "../rules/index.js";
3
+ import { buildRuleFilter } from "./page-filter.js";
4
+ import { buildPageSource, PAGE_SOURCE } from "./page-source.js";
5
+ /**
6
+ * How many urls to keep per group to pick the examples from. The sample is
7
+ * ordered by a hash of the creation time, so it is arbitrary but stable, and
8
+ * three distinct urls come out of it comfortably.
9
+ */
10
+ const EXAMPLE_SAMPLE = 50;
11
+ /**
12
+ * The query parameters that appear on accelerated pages, ranked by how many
13
+ * cache entries stripping them would collapse.
14
+ *
15
+ * Ranked *per parameter*: a parameter travels in many different combinations —
16
+ * on a large shop a single one shows up in over a thousand of them — so a
17
+ * ranking over whole combinations never states what one parameter is worth,
18
+ * which is the question being asked. `bySet` brings the combination view back.
19
+ *
20
+ * Parameters the config already strips are filtered out, so the result only
21
+ * lists candidates that still need a decision.
22
+ */
23
+ export function buildParameterQuery(input) {
24
+ const { cte, subjects } = buildPageSource(input, { columns: ["createdat"] });
25
+ const filter = buildRuleFilter(input, subjects);
26
+ const warnings = [...filter.warnings];
27
+ const stripParameters = prepareStripParameters(input.ruleSets.stripQueryParams, subjects, { hosts: input.hosts });
28
+ warnings.push(...stripParameters.warnings);
29
+ const kept = notStrippedParameter(stripParameters, raw("key"), warnings);
30
+ // Only urls that carry a query string can contribute a candidate.
31
+ const rows = and(filter.predicate, greaterThan(call("cardinality", call("split", column("url"), literal("?"))), literal(1)));
32
+ const sql = `
33
+ with ${cte}, urlWithParams as (
34
+ select
35
+ url,
36
+ createdat,
37
+ split(url, '?')[1] as path,
38
+ split(split(url, '?')[2], '&') as pairs,
39
+ filter(
40
+ transform(
41
+ split(split(url, '?')[2], '&'),
42
+ (pair) -> split(pair, '=')[1]
43
+ ),
44
+ (key) -> ${formatPredicate(kept, { level: 4 })}
45
+ ) as keptKeys
46
+ from ${PAGE_SOURCE}
47
+ where ${formatPredicate(rows, { level: 2 })}
48
+ ), cacheKeys as (
49
+ select
50
+ url,
51
+ createdat,
52
+ path,
53
+ pairs,
54
+ path || '?' || array_join(pairs, '&') as cacheKey
55
+ from (
56
+ select
57
+ url,
58
+ createdat,
59
+ path,
60
+ ${keptPairs()} as pairs
61
+ from urlWithParams
62
+ )
63
+ where cardinality(pairs) > 0
64
+ )
65
+
66
+ ${input.bySet ? bySetQuery(input) : byParameterQuery(input)}`;
67
+ return {
68
+ approximated: filter.approximated,
69
+ dropped: [...filter.dropped, ...stripParameters.dropped],
70
+ findings: filter.findings,
71
+ reduced: filter.reduced,
72
+ sql,
73
+ warnings,
74
+ };
75
+ }
76
+ /**
77
+ * The parameters of a url that Speed Kit keeps, sorted and deduplicated.
78
+ *
79
+ * Sorting matters twice over: it makes `?b=1&a=1` and `?a=1&b=1` one cache key
80
+ * rather than two — which is what Speed Kit stores, since it sorts the query
81
+ * string — and it stops one set of parameters from being counted as several
82
+ * different ones.
83
+ */
84
+ function keptPairs() {
85
+ return `array_sort(
86
+ array_distinct(
87
+ filter(pairs, (pair) -> contains(keptKeys, split(pair, '=')[1]))
88
+ )
89
+ )`;
90
+ }
91
+ /** One row per parameter: what it costs, and what dropping it would save. */
92
+ function byParameterQuery(input) {
93
+ return `select
94
+ parameter,
95
+ count(*) as PIs,
96
+ count(distinct value) as distinctValues,
97
+ count(distinct url) as urls,
98
+ count(distinct cacheKey) as cachedUrls,
99
+ count(distinct cacheKey) - count(distinct cacheKeyWithout) as potentialGain,
100
+ ${example(1)} as example1,
101
+ ${example(2)} as example2,
102
+ ${example(3)} as example3
103
+ from (
104
+ select
105
+ url,
106
+ createdat,
107
+ cacheKey,
108
+ split(pair, '=')[1] as parameter,
109
+ if(cardinality(split(pair, '=')) > 1, split(pair, '=')[2], '') as value,
110
+ path || if(
111
+ cardinality(filter(pairs, (other) -> other <> pair)) = 0,
112
+ '',
113
+ '?' || array_join(filter(pairs, (other) -> other <> pair), '&')
114
+ ) as cacheKeyWithout
115
+ from cacheKeys
116
+ cross join unnest(pairs) as t(pair)
117
+ )
118
+ group by parameter${having(input)}
119
+ order by potentialGain desc, PIs desc, cachedUrls desc`;
120
+ }
121
+ /** One row per combination of parameters, the way the query read before. */
122
+ function bySetQuery(input) {
123
+ return `select
124
+ '[' || array_join(transform(pairs, (pair) -> split(pair, '=')[1]), ', ') || ']' as paramKeys,
125
+ count(distinct cacheKey) - count(distinct path) as potentialGain,
126
+ count(distinct cacheKey) as cachedUrls,
127
+ count(distinct url) as urls,
128
+ count(*) as PIs,
129
+ ${example(1)} as example1,
130
+ ${example(2)} as example2,
131
+ ${example(3)} as example3
132
+ from cacheKeys
133
+ group by pairs${having(input)}
134
+ order by potentialGain desc, cachedUrls desc, PIs desc`;
135
+ }
136
+ function having(input) {
137
+ const minimum = Math.max(1, Math.floor(input.minImpressions ?? 1));
138
+ return minimum > 1 ? `\nhaving count(*) >= ${minimum}` : "";
139
+ }
140
+ function example(index) {
141
+ return `element_at(array_distinct(max_by(url, MD5(to_utf8(to_iso8601(createdat))), ${EXAMPLE_SAMPLE})), ${index})`;
142
+ }
@@ -0,0 +1,27 @@
1
+ import { AthenaParameter } from "../../athena/index.js";
2
+ export interface PopsQueryInput {
3
+ readonly app: string;
4
+ /** Size of the window the query looks at, in days. Defaults to a week. */
5
+ readonly days?: number;
6
+ /** The day the window is measured from. Defaults to now. */
7
+ readonly now?: Date;
8
+ /**
9
+ * Count page-impression beacons instead of Speed Kit's own asset requests.
10
+ * Needed before go-live, when there is no accelerated traffic yet.
11
+ */
12
+ readonly useRum?: boolean;
13
+ }
14
+ export interface PopsQuery {
15
+ /** The values for the query's placeholders, in order. */
16
+ readonly parameters: AthenaParameter[];
17
+ readonly sql: string;
18
+ }
19
+ /**
20
+ * Which Fastly PoPs serve an app's traffic, per origin and region.
21
+ *
22
+ * The server pre-warms the PoPs relevant for a customer whenever a document is
23
+ * stored; this is what tells it which ones those are. Unlike the pre-warm and
24
+ * parameter queries it is not derived from the Speed Kit config — the log
25
+ * already knows which requests belong to Speed Kit — so it takes no rules.
26
+ */
27
+ export declare function buildPopsQuery(input: PopsQueryInput): PopsQuery;
@@ -0,0 +1,65 @@
1
+ import { and, between, booleanColumn, call, column, concat, equals, formatPredicate, isNull, literal, notInValues, parameter, } from "../sql/index.js";
2
+ import { DEFAULT_QUERY_DAYS, sinceDate, today } from "./query-model.js";
3
+ /** The relation carrying Fastly's request log. */
4
+ const FASTLY_LOGS = "fastly.logs";
5
+ /** The common table expression the aggregation reads from. */
6
+ const REQUESTS = "requests";
7
+ /**
8
+ * The locale segment of a path, e.g. `/de/`, `/de-at/`, `/shop/de/`.
9
+ *
10
+ * Origins are configured per market, so the prefix belongs to the origin the
11
+ * request was served for — `https://www.shop.com/de/` is a different origin
12
+ * than `https://www.shop.com/fr/`.
13
+ */
14
+ const LOCALE_PREFIX = String.raw `(?i)((/\w+)?/[a-z]{2}([-/_][a-z]{2})*/)`;
15
+ /**
16
+ * Which Fastly PoPs serve an app's traffic, per origin and region.
17
+ *
18
+ * The server pre-warms the PoPs relevant for a customer whenever a document is
19
+ * stored; this is what tells it which ones those are. Unlike the pre-warm and
20
+ * parameter queries it is not derived from the Speed Kit config — the log
21
+ * already knows which requests belong to Speed Kit — so it takes no rules.
22
+ */
23
+ export function buildPopsQuery(input) {
24
+ const days = input.days ?? DEFAULT_QUERY_DAYS.pops;
25
+ const sql = `
26
+ with ${REQUESTS} as (
27
+ select
28
+ region,
29
+ pop,
30
+ origin,
31
+ ${originWithPrefix()} as originWithPrefix
32
+ from ${FASTLY_LOGS}
33
+ where ${formatPredicate(requestFilter(input), { level: 2 })}
34
+ )
35
+
36
+ select
37
+ region,
38
+ pop,
39
+ coalesce(originWithPrefix, origin) as origin,
40
+ count() as hits
41
+ from ${REQUESTS}
42
+ group by grouping sets (
43
+ (region, pop, origin),
44
+ (region, pop, origin, originWithPrefix)
45
+ )
46
+ having originWithPrefix is null or originWithPrefix <> 'Unknown'`;
47
+ return {
48
+ // The order has to match the placeholders in the printed SQL.
49
+ parameters: [sinceDate(days, input.now), today(input.now), input.app],
50
+ sql,
51
+ };
52
+ }
53
+ /** The origin a request was served for, including its locale prefix. */
54
+ function originWithPrefix() {
55
+ const assetUrl = column("asseturl");
56
+ return call("coalesce", concat(literal("https://"), call("url_extract_host", assetUrl), call("regexp_extract", call("url_extract_path", assetUrl), literal(LOCALE_PREFIX), literal(0))), literal("Unknown"));
57
+ }
58
+ function requestFilter(input) {
59
+ return and(between(column("date"), parameter(), parameter()), equals(column("app"), parameter()),
60
+ // Before go-live there are no Speed Kit asset requests to count, so the
61
+ // beacons the snippet sends stand in for them.
62
+ input.useRum
63
+ ? booleanColumn("ispibeacon")
64
+ : and(booleanColumn("isassetrequest"), booleanColumn("isnavigaterequest")), isNull(column("client.bot")), notInValues(column("origin"), ["Unknown", ""]));
65
+ }
@@ -0,0 +1,18 @@
1
+ import { BuiltQuery, QueryInput } from "./query-model.js";
2
+ /**
3
+ * The URLs to pre-warm, most requested first: every page Speed Kit
4
+ * accelerates, reduced to the cache key it would be stored under.
5
+ *
6
+ * Real traffic has a long tail — two thirds of the URLs of a ten day window
7
+ * are typically seen exactly once — and every URL in the list costs the origin
8
+ * one request per variation when the job runs. `minHits` and `limit` are there
9
+ * to cut that tail; `hits` is selected so a list can be judged before it is
10
+ * used.
11
+ *
12
+ * A list can carry the variation to warm each URL in, rather than leaving the
13
+ * pre-warmer to request every URL in every variation: `devices` reads it off
14
+ * the traffic, `variations` writes the caller's own set into it. Neither takes
15
+ * anything away from the URL list itself — `minHits`, `limit` and the 404 rule
16
+ * keep judging URLs, so `--limit 5000` is still 5000 pages.
17
+ */
18
+ export declare function buildPrewarmQuery(input: QueryInput): BuiltQuery;
@@ -0,0 +1,227 @@
1
+ import { and, call, column, formatPredicate, greaterOrEqual, isTrue, leaf, literal, lower, raw, TRUE, } from "../sql/index.js";
2
+ import { prepareStripParameters, strippedUrlExpression, } from "../rules/index.js";
3
+ import { normalizeDevices } from "./device.js";
4
+ import { buildRuleFilter } from "./page-filter.js";
5
+ import { buildPageSource, PAGE_SOURCE } from "./page-source.js";
6
+ /** The response cause Speed Kit records when the origin answered with a 404. */
7
+ const NOT_FOUND = "OriginStatusNotFound";
8
+ /** Two spaces per level, as everywhere in the emitted queries. */
9
+ const INDENT = " ";
10
+ /** The name of the common table expression holding one row per impression. */
11
+ const IMPRESSIONS = "impressions";
12
+ /** The name of the common table expression holding one row per url. */
13
+ const URLS = "urls";
14
+ /** The name of the common table expression holding one row per url and variation. */
15
+ const PAIRS = "pairs";
16
+ /**
17
+ * The URLs to pre-warm, most requested first: every page Speed Kit
18
+ * accelerates, reduced to the cache key it would be stored under.
19
+ *
20
+ * Real traffic has a long tail — two thirds of the URLs of a ten day window
21
+ * are typically seen exactly once — and every URL in the list costs the origin
22
+ * one request per variation when the job runs. `minHits` and `limit` are there
23
+ * to cut that tail; `hits` is selected so a list can be judged before it is
24
+ * used.
25
+ *
26
+ * A list can carry the variation to warm each URL in, rather than leaving the
27
+ * pre-warmer to request every URL in every variation: `devices` reads it off
28
+ * the traffic, `variations` writes the caller's own set into it. Neither takes
29
+ * anything away from the URL list itself — `minHits`, `limit` and the 404 rule
30
+ * keep judging URLs, so `--limit 5000` is still 5000 pages.
31
+ */
32
+ export function buildPrewarmQuery(input) {
33
+ const devices = normalizeDevices(input.devices);
34
+ const { cte, subjects } = buildPageSource(input, {
35
+ columns: input.excludeNotFound ? ["responsecause"] : [],
36
+ });
37
+ const filter = buildRuleFilter(input, subjects);
38
+ const warnings = [...filter.warnings];
39
+ const stripParameters = prepareStripParameters(input.ruleSets.stripQueryParams, subjects, { hosts: input.hosts });
40
+ warnings.push(...stripParameters.warnings);
41
+ const cacheKey = tidyQueryString(strippedUrlExpression(stripParameters, column("url"), warnings));
42
+ const impressions = impressionSelect(input, cacheKey, filter.predicate, {
43
+ byDevice: devices.length > 0,
44
+ });
45
+ return {
46
+ approximated: filter.approximated,
47
+ dropped: [...filter.dropped, ...stripParameters.dropped],
48
+ findings: filter.findings,
49
+ reduced: filter.reduced,
50
+ sql: assemble(input, cte, impressions, devices.length > 0),
51
+ warnings,
52
+ };
53
+ }
54
+ /**
55
+ * The three shapes of the list, and the count of whichever one was asked for.
56
+ * Without any of the flags the query is the one it always was, down to the
57
+ * byte — every one of them is opt-in, and the checked-in golden queries are
58
+ * the proof of it.
59
+ */
60
+ function assemble(input, pages, impressions, byDevice) {
61
+ if (input.count) {
62
+ return assembleCount(input, pages, impressions, byDevice);
63
+ }
64
+ if (byDevice) {
65
+ // Ordered by the pair's hits, so the variations interleave: a run that is
66
+ // aborted or rate-limited into the ground has warmed the busiest entries
67
+ // of every device class, not all of one and none of the next.
68
+ return `
69
+ with ${pages},
70
+ ${IMPRESSIONS} as (
71
+ ${impressions}
72
+ ),
73
+ ${URLS} as (
74
+ ${keptUrlSelect(input, IMPRESSIONS, 1)}
75
+ )
76
+
77
+ ${pairSelect({ ordered: true })}`;
78
+ }
79
+ const variations = input.variations ?? [];
80
+ if (variations.length > 0) {
81
+ // Nothing says which url carries which custom variation, so every url gets
82
+ // every one of them. Ordering can only be url-major here.
83
+ return `
84
+ with ${pages},
85
+ ${IMPRESSIONS} as (
86
+ ${impressions}
87
+ ),
88
+ ${URLS} as (
89
+ ${keptUrlSelect(input, IMPRESSIONS, 1)}
90
+ )
91
+
92
+ select ${URLS}.url, variation, ${URLS}.hits
93
+ from ${URLS}
94
+ cross join unnest(array[${variations.map((variation) => literal(variation)).join(", ")}]) as t (variation)
95
+ order by ${URLS}.hits desc, variation`;
96
+ }
97
+ return `
98
+ with ${pages}
99
+
100
+ ${keptUrlSelect(input, `(\n${impressions}\n)`)}`;
101
+ }
102
+ /**
103
+ * How large the list is, instead of what is in it.
104
+ *
105
+ * Every cut the list makes is still made — the same relations are built from
106
+ * the same options, only the final projection counts them instead of
107
+ * selecting them. So a count of 4200 is the promise that the list is 4200 urls
108
+ * long, and a `--limit` shows up as the count stopping at the limit.
109
+ *
110
+ * `impressions` is what those urls stood for in the window: it says whether a
111
+ * `--minHits` cut away a tail or a third of the traffic.
112
+ */
113
+ function assembleCount(input, pages, impressions, byDevice) {
114
+ // All three shapes count the same relations, so they are built once —
115
+ // unlike the list itself, a count has no reason to inline anything.
116
+ const relations = `
117
+ with ${pages},
118
+ ${IMPRESSIONS} as (
119
+ ${impressions}
120
+ ),
121
+ ${URLS} as (
122
+ ${keptUrlSelect(input, IMPRESSIONS, 1)}
123
+ )`;
124
+ if (byDevice) {
125
+ // Counted over the pairs, not over the urls: the total is what the
126
+ // pre-warmer would request, which is every url once per variation it was
127
+ // seen in — and that is more than there are urls.
128
+ //
129
+ // The column is qualified throughout because the projection names its
130
+ // result `variation` too, and an unqualified `grouping(variation)` in the
131
+ // order by resolves to that output alias, which Athena rejects.
132
+ return `${relations},
133
+ ${PAIRS} as (
134
+ ${pairSelect({ level: 1 })}
135
+ )
136
+
137
+ select coalesce(${PAIRS}.variation, 'total') as variation, count(*) as urls, sum(hits) as impressions
138
+ from ${PAIRS}
139
+ group by rollup(${PAIRS}.variation)
140
+ order by grouping(${PAIRS}.variation) desc, 2 desc`;
141
+ }
142
+ // Every url carries every custom variation, so a row per variation would be
143
+ // the same number written out n times. The product is the row instead.
144
+ const variations = input.variations ?? [];
145
+ const projections = [
146
+ "count(*) as urls",
147
+ ...(variations.length > 0
148
+ ? [`${variations.length} * count(*) as requests`]
149
+ : []),
150
+ "sum(hits) as impressions",
151
+ ];
152
+ return `${relations}
153
+
154
+ select ${projections.join(", ")}
155
+ from ${URLS}`;
156
+ }
157
+ /**
158
+ * One row per url and variation, for a list split by device class: the urls
159
+ * the list keeps, each paired with the variations it was requested in.
160
+ */
161
+ function pairSelect(options = {}) {
162
+ const indent = INDENT.repeat(options.level ?? 0);
163
+ return [
164
+ `${indent}select ${IMPRESSIONS}.url, ${IMPRESSIONS}.variation, count(*) as hits`,
165
+ `${indent}from ${IMPRESSIONS}`,
166
+ `${indent}${INDENT}join ${URLS} on ${URLS}.url = ${IMPRESSIONS}.url`,
167
+ `${indent}group by 1, 2`,
168
+ ...(options.ordered ? [`${indent}order by 3 desc`] : []),
169
+ ].join("\n");
170
+ }
171
+ /**
172
+ * One row per page impression, reduced to the cache key its url would be
173
+ * stored under — and, for a split list, the variation it was requested in.
174
+ */
175
+ function impressionSelect(input, cacheKey, predicate, options) {
176
+ const projections = [
177
+ `${cacheKey} as url`,
178
+ ...(options.byDevice ? [`${lower(column("device"))} as variation`] : []),
179
+ ...(input.excludeNotFound ? [column("responsecause")] : []),
180
+ ];
181
+ return [
182
+ `${INDENT}select`,
183
+ `${INDENT.repeat(2)}${projections.join(`,\n${INDENT.repeat(2)}`)}`,
184
+ `${INDENT}from ${PAGE_SOURCE}`,
185
+ `${INDENT}where ${formatPredicate(predicate, { level: 2 })}`,
186
+ ].join("\n");
187
+ }
188
+ /**
189
+ * The urls worth pre-warming, most requested first — the whole query when
190
+ * there is no split, and the relation the split is judged against when there
191
+ * is one. Either way it counts urls, not rows.
192
+ */
193
+ function keptUrlSelect(input, source, level = 0) {
194
+ const kept = keptUrls(input);
195
+ const indent = INDENT.repeat(level);
196
+ return [
197
+ `${indent}select url, count(*) as hits`,
198
+ `${indent}from ${source}`,
199
+ `${indent}group by url`,
200
+ ...(kept
201
+ ? [`${indent}having ${formatPredicate(kept, { level: level + 1 })}`]
202
+ : []),
203
+ `${indent}order by hits desc`,
204
+ ...(input.limit
205
+ ? [`${indent}limit ${Math.max(1, Math.floor(input.limit))}`]
206
+ : []),
207
+ ].join("\n");
208
+ }
209
+ /** Which of the grouped urls are worth pre-warming. */
210
+ function keptUrls(input) {
211
+ const minHits = Math.max(1, Math.floor(input.minHits ?? 1));
212
+ const predicate = and(minHits > 1 ? greaterOrEqual(raw("count(*)"), literal(minHits)) : TRUE,
213
+ // A url the origin answers with a 404 more often than not is gone; only
214
+ // the majority decides, because a single 404 among real hits says nothing.
215
+ input.excludeNotFound
216
+ ? leaf(`2 * count_if(responsecause = ${literal(NOT_FOUND)}) < count(*)`)
217
+ : TRUE);
218
+ return isTrue(predicate) ? undefined : predicate;
219
+ }
220
+ /**
221
+ * Repairs the URL after the stripped parameters were cut out of it: a query
222
+ * string that lost its first parameter starts with `&`, and one that lost all
223
+ * of them ends in a bare `?`.
224
+ */
225
+ function tidyQueryString(url) {
226
+ return call("regexp_replace", call("regexp_replace", url, literal("(^[^&?]*)&"), literal("$1?")), literal("\\?$"), literal(""));
227
+ }
@@ -0,0 +1,97 @@
1
+ import { ParsedRuleSets, RuleNote } from "../rules/index.js";
2
+ import { DeviceVariation } from "./device.js";
3
+ /** How far back a query looks, per query type. */
4
+ export declare const DEFAULT_QUERY_DAYS: {
5
+ readonly parameter: 30;
6
+ readonly pops: 7;
7
+ readonly prewarm: 10;
8
+ };
9
+ /** Options that shape the pre-warm query's result. */
10
+ export interface PrewarmOptions {
11
+ /**
12
+ * Report how large the list is instead of what is in it: one row of counts,
13
+ * one per variation where the list carries variations.
14
+ *
15
+ * Every other option still applies — the count is the size of the list the
16
+ * same command would produce, `--limit` and `--minHits` included.
17
+ */
18
+ readonly count?: boolean;
19
+ /**
20
+ * Split the list by device class, so every row carries the variation its
21
+ * url was actually requested in. Mutually exclusive with {@link variations}.
22
+ *
23
+ * A device class not named here is dropped: a url only ever seen on a
24
+ * tablet produces no row when tablets are not asked for.
25
+ */
26
+ readonly devices?: readonly DeviceVariation[];
27
+ /** Keep only the most requested urls. */
28
+ readonly limit?: number;
29
+ /**
30
+ * Drop urls whose page impressions are mostly answered by an origin 404 —
31
+ * pre-warming them fetches a page that is not there.
32
+ */
33
+ readonly excludeNotFound?: boolean;
34
+ /** Keep only urls requested at least this often in the window. */
35
+ readonly minHits?: number;
36
+ /**
37
+ * Keep only urls matching one of these regular expressions, on top of what
38
+ * the config already decides. The subject is the url a rule sees — without
39
+ * scheme and fragment, e.g. `www.shop.de/p/1`.
40
+ */
41
+ readonly match?: readonly string[];
42
+ /**
43
+ * Pair every url of the list with each of these variations. Mutually
44
+ * exclusive with {@link devices}.
45
+ *
46
+ * No data says which url carries which custom variation, so this is the
47
+ * caller's guess, written into the list instead of applied to it later.
48
+ */
49
+ readonly variations?: readonly string[];
50
+ }
51
+ /** Options that shape the parameter query's result. */
52
+ export interface ParameterOptions {
53
+ /**
54
+ * Rank whole sets of parameters, the way the query did before there was a
55
+ * per-parameter ranking. Useful to see which parameters travel together.
56
+ */
57
+ readonly bySet?: boolean;
58
+ /** Leave out parameters seen on fewer page impressions than this. */
59
+ readonly minImpressions?: number;
60
+ }
61
+ /** Everything a query is built from. */
62
+ export interface QueryInput extends ParameterOptions, PrewarmOptions {
63
+ readonly app: string;
64
+ /** Size of the window the query looks at, in days. */
65
+ readonly days: number;
66
+ /** Drop page impressions of pages marked `noindex`. */
67
+ readonly excludeNoindex?: boolean;
68
+ /** Drop page impressions Speed Kit classified as suspicious. */
69
+ readonly excludeSuspicious?: boolean;
70
+ /** Keep hard navigations only, dropping soft (SPA) navigations. */
71
+ readonly hardNavigationsOnly?: boolean;
72
+ /** Hosts of the app's origins, used to recognise rules that cannot fire. */
73
+ readonly hosts?: readonly string[];
74
+ /** The day the window is measured from. Defaults to now. */
75
+ readonly now?: Date;
76
+ readonly ruleSets: ParsedRuleSets;
77
+ }
78
+ export interface BuiltQuery {
79
+ /** Rules whose emitted filter is wider than the rule itself. */
80
+ readonly approximated: RuleNote[];
81
+ /** Rules left out because they can never apply to a page impression. */
82
+ readonly dropped: RuleNote[];
83
+ /** Findings about the config the reduction turned up. */
84
+ readonly findings: RuleNote[];
85
+ /** Rules left out because another rule already decides their pages. */
86
+ readonly reduced: RuleNote[];
87
+ readonly sql: string;
88
+ /** Patterns whose meaning may differ between the browser and Athena. */
89
+ readonly warnings: string[];
90
+ }
91
+ /** Today, as `YYYY-MM-DD` — the format of the `date` partition column. */
92
+ export declare function today(now?: Date): string;
93
+ /**
94
+ * The day the query starts at, `days` before now, as `YYYY-MM-DD` — the
95
+ * format of the `date` partition column.
96
+ */
97
+ export declare function sinceDate(days: number, now?: Date): string;
@@ -0,0 +1,19 @@
1
+ /** How far back a query looks, per query type. */
2
+ export const DEFAULT_QUERY_DAYS = {
3
+ parameter: 30,
4
+ pops: 7,
5
+ prewarm: 10,
6
+ };
7
+ /** Today, as `YYYY-MM-DD` — the format of the `date` partition column. */
8
+ export function today(now = new Date()) {
9
+ return now.toISOString().slice(0, 10);
10
+ }
11
+ /**
12
+ * The day the query starts at, `days` before now, as `YYYY-MM-DD` — the
13
+ * format of the `date` partition column.
14
+ */
15
+ export function sinceDate(days, now = new Date()) {
16
+ const window = Math.max(1, Math.floor(days));
17
+ const since = new Date(now.getTime() - window * 24 * 60 * 60 * 1000);
18
+ return since.toISOString().slice(0, 10);
19
+ }
@@ -0,0 +1,18 @@
1
+ /**
2
+ * Config shapes the query builder has to get right, each one standing for a
3
+ * class of real customer configs.
4
+ *
5
+ * These feed the golden queries in `golden.spec.ts`: the whole emitted SQL and
6
+ * the whole accounting are checked in next to them, so any change to the
7
+ * builder shows up as a reviewable diff instead of as a surprise in a
8
+ * pre-warm run.
9
+ */
10
+ export interface GoldenConfig {
11
+ /** The evaluated `window.speedKit` of a customer config. */
12
+ readonly config: Record<string, unknown>;
13
+ /** Why this shape is here. */
14
+ readonly describes: string;
15
+ readonly hosts: readonly string[];
16
+ readonly name: string;
17
+ }
18
+ export declare const GOLDEN_CONFIGS: readonly GoldenConfig[];