@speedkit/cli 4.44.0-beta.4 → 4.44.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/CHANGELOG.md +35 -13
  2. package/README.md +68 -42
  3. package/dist/base/sk-command.d.ts +7 -0
  4. package/dist/base/sk-command.js +19 -0
  5. package/dist/commands/deploy.d.ts +1 -0
  6. package/dist/commands/deploy.js +5 -1
  7. package/dist/commands/diff.d.ts +47 -0
  8. package/dist/commands/diff.js +67 -0
  9. package/dist/commands/generate-pop-config.js +3 -7
  10. package/dist/helpers/race.js +4 -2
  11. package/dist/helpers/spec/race.spec.js +31 -0
  12. package/dist/models/files.d.ts +0 -1
  13. package/dist/models/files.js +0 -1
  14. package/dist/services/browser-fetch/browser-origin-fetch.js +1 -1
  15. package/dist/services/browser-fetch/browser-origin-fetcher.d.ts +1 -7
  16. package/dist/services/browser-fetch/browser-origin-fetcher.js +3 -26
  17. package/dist/services/bundler/bundle-service-factory.d.ts +0 -3
  18. package/dist/services/bundler/bundle-service-factory.js +1 -5
  19. package/dist/services/bundler/bundle-service.d.ts +1 -10
  20. package/dist/services/bundler/bundle-service.js +1 -16
  21. package/dist/services/bundler/index.d.ts +0 -1
  22. package/dist/services/bundler/index.js +0 -1
  23. package/dist/services/cli/cli-environment.d.ts +11 -0
  24. package/dist/services/cli/cli-environment.js +37 -0
  25. package/dist/services/cli/cli-service-model.d.ts +3 -1
  26. package/dist/services/cli/cli-service.d.ts +1 -1
  27. package/dist/services/cli/cli-service.js +4 -1
  28. package/dist/services/cli/spec/cli-environment.spec.js +30 -1
  29. package/dist/services/deploy/checks/activation/customer-approval-check.js +1 -1
  30. package/dist/services/deploy/checks/post-deploy/refresh-deployment-job-check.d.ts +2 -2
  31. package/dist/services/deploy/checks/post-deploy/refresh-deployment-job-check.js +8 -7
  32. package/dist/services/deploy/checks/post-deploy/revalidated-module-check.d.ts +2 -2
  33. package/dist/services/deploy/checks/post-deploy/revalidated-module-check.js +11 -7
  34. package/dist/services/deploy/checks/post-deploy/save-files-check.d.ts +2 -2
  35. package/dist/services/deploy/checks/post-deploy/save-files-check.js +3 -2
  36. package/dist/services/deploy/checks/pre-deploy/device-detection-changed-check.js +3 -2
  37. package/dist/services/deploy/checks/pre-deploy/split-change-check.js +3 -2
  38. package/dist/services/deploy/deploy-service-build-context.d.ts +2 -0
  39. package/dist/services/deploy/deploy-service-build-context.js +13 -2
  40. package/dist/services/deploy/deploy-service-factory.js +2 -9
  41. package/dist/services/deploy/deploy-service-model.d.ts +11 -1
  42. package/dist/services/deploy/deploy-service.d.ts +2 -0
  43. package/dist/services/deploy/deploy-service.js +26 -1
  44. package/dist/services/deploy/handler/customer-config-handler.d.ts +2 -2
  45. package/dist/services/deploy/handler/install-resource-handler.d.ts +2 -2
  46. package/dist/services/deploy/handler/install-resource-handler.js +6 -3
  47. package/dist/services/deploy/handler/module-handler.d.ts +2 -2
  48. package/dist/services/deploy/handler/module-handler.js +2 -2
  49. package/dist/services/deploy/handler/server-config-handler.d.ts +2 -2
  50. package/dist/services/deploy/handler/server-config-handler.js +2 -2
  51. package/dist/services/deploy/mr/live-state-check.d.ts +2 -2
  52. package/dist/services/deploy/mr/merge-request-deploy-service-factory.js +2 -7
  53. package/dist/services/deploy/policy/deploy-policy-model.d.ts +1 -5
  54. package/dist/services/deploy/policy/deploy-policy-model.js +0 -4
  55. package/dist/services/deploy/policy/index.d.ts +1 -0
  56. package/dist/services/deploy/policy/index.js +1 -0
  57. package/dist/services/deploy/policy/testing-deploy-policy.d.ts +16 -0
  58. package/dist/services/deploy/policy/testing-deploy-policy.js +42 -0
  59. package/dist/services/deploy/spec/checks/post-deploy/refresh-deployment-job-check.spec.js +5 -12
  60. package/dist/services/deploy/spec/deploy-service.spec.js +64 -0
  61. package/dist/services/deploy/spec/handler/install-resource-handler.spec.js +58 -0
  62. package/dist/services/deploy/spec/mr/live-state-check.spec.js +2 -2
  63. package/dist/services/deploy/spec/policy/auto-deploy-policy.spec.js +0 -2
  64. package/dist/services/deploy/spec/policy/testing-deploy-policy.spec.d.ts +1 -0
  65. package/dist/services/deploy/spec/policy/testing-deploy-policy.spec.js +46 -0
  66. package/dist/services/diff/create-diff-engine.d.ts +14 -0
  67. package/dist/services/diff/create-diff-engine.js +42 -0
  68. package/dist/services/diff/customer-origin.d.ts +9 -0
  69. package/dist/services/diff/customer-origin.js +19 -0
  70. package/dist/services/diff/diff-engine.d.ts +30 -0
  71. package/dist/services/diff/diff-engine.js +49 -0
  72. package/dist/services/diff/diff-log.d.ts +9 -0
  73. package/dist/services/diff/diff-log.js +18 -0
  74. package/dist/services/diff/diff-model.d.ts +68 -0
  75. package/dist/services/diff/diff-model.js +1 -0
  76. package/dist/services/diff/diff-service-build-context.d.ts +99 -9
  77. package/dist/services/diff/diff-service-build-context.js +299 -1
  78. package/dist/services/diff/diff-service-factory.d.ts +24 -2
  79. package/dist/services/diff/diff-service-factory.js +165 -3
  80. package/dist/services/diff/diff-service.d.ts +93 -33
  81. package/dist/services/diff/diff-service.js +274 -140
  82. package/dist/services/diff/index.d.ts +12 -1
  83. package/dist/services/diff/index.js +12 -1
  84. package/dist/services/diff/noise-baseline.d.ts +28 -0
  85. package/dist/services/diff/noise-baseline.js +105 -0
  86. package/dist/services/diff/page-source.d.ts +42 -0
  87. package/dist/services/diff/page-source.js +93 -0
  88. package/dist/services/diff/side-failure.d.ts +28 -0
  89. package/dist/services/diff/side-failure.js +100 -0
  90. package/dist/services/diff/sinks/json-diff-sink.d.ts +32 -0
  91. package/dist/services/diff/sinks/json-diff-sink.js +44 -0
  92. package/dist/services/diff/sinks/no-diff-sink.d.ts +5 -0
  93. package/dist/services/diff/sinks/no-diff-sink.js +6 -0
  94. package/dist/services/diff/sinks/patch-diff-sink.d.ts +19 -0
  95. package/dist/services/diff/sinks/patch-diff-sink.js +29 -0
  96. package/dist/services/diff/sinks/tool-diff-sink.d.ts +26 -0
  97. package/dist/services/diff/sinks/tool-diff-sink.js +88 -0
  98. package/dist/services/diff/sinks/unified-patch.d.ts +7 -0
  99. package/dist/services/diff/sinks/unified-patch.js +12 -0
  100. package/dist/services/diff/spec/customer-origin.spec.d.ts +1 -0
  101. package/dist/services/diff/spec/customer-origin.spec.js +27 -0
  102. package/dist/services/diff/spec/diff-engine.spec.d.ts +1 -0
  103. package/dist/services/diff/spec/diff-engine.spec.js +47 -0
  104. package/dist/services/diff/spec/diff-service-build-context.spec.d.ts +1 -0
  105. package/dist/services/diff/spec/diff-service-build-context.spec.js +185 -0
  106. package/dist/services/diff/spec/diff-service.spec.d.ts +1 -0
  107. package/dist/services/diff/spec/diff-service.spec.js +393 -0
  108. package/dist/services/diff/spec/noise-baseline.spec.d.ts +1 -0
  109. package/dist/services/diff/spec/noise-baseline.spec.js +95 -0
  110. package/dist/services/diff/spec/page-source.spec.d.ts +1 -0
  111. package/dist/services/diff/spec/page-source.spec.js +140 -0
  112. package/dist/services/diff/spec/sinks/json-diff-sink.spec.d.ts +1 -0
  113. package/dist/services/diff/spec/sinks/json-diff-sink.spec.js +72 -0
  114. package/dist/services/diff/spec/sinks/patch-diff-sink.spec.d.ts +1 -0
  115. package/dist/services/diff/spec/sinks/patch-diff-sink.spec.js +47 -0
  116. package/dist/services/diff/spec/sinks/tool-diff-sink.spec.d.ts +1 -0
  117. package/dist/services/diff/spec/sinks/tool-diff-sink.spec.js +61 -0
  118. package/dist/services/document-handler-runtime/document-handler-abort.d.ts +2 -0
  119. package/dist/services/document-handler-runtime/document-handler-abort.js +2 -0
  120. package/dist/services/document-handler-runtime/document-handler-server.d.ts +0 -17
  121. package/dist/services/document-handler-runtime/document-handler-server.js +19 -81
  122. package/dist/services/document-handler-runtime/runtime-files.d.ts +9 -0
  123. package/dist/services/document-handler-runtime/runtime-files.js +17 -0
  124. package/dist/services/integration-api/integration-api-model.d.ts +1 -2
  125. package/dist/services/integration-api/spec/speed-kit-config-file.spec.js +8 -68
  126. package/dist/services/integration-api/struct/customer-config.d.ts +1 -32
  127. package/dist/services/integration-api/struct/customer-config.js +3 -33
  128. package/dist/services/integration-api/struct/speed-kit-config-file.d.ts +4 -9
  129. package/dist/services/integration-api/struct/speed-kit-config-file.js +10 -27
  130. package/dist/services/onboarding/config-renderer/speed-kit-install-context.js +8 -25
  131. package/dist/services/onboarding/dashboard/diff-against-current-page.d.ts +29 -7
  132. package/dist/services/onboarding/dashboard/diff-against-current-page.js +24 -37
  133. package/dist/services/onboarding/dashboard/index.js +13 -6
  134. package/dist/services/onboarding/dashboard/request-diff-service.d.ts +21 -8
  135. package/dist/services/onboarding/dashboard/request-diff-service.js +21 -49
  136. package/dist/services/onboarding/onboarding-model.d.ts +0 -1
  137. package/dist/services/onboarding/onboarding-model.js +0 -1
  138. package/dist/services/onboarding/onboarding-service-factory.d.ts +0 -1
  139. package/dist/services/onboarding/onboarding-service-factory.js +24 -30
  140. package/dist/services/onboarding/spec/dashboard/request-diff-service.spec.d.ts +1 -0
  141. package/dist/services/onboarding/spec/dashboard/request-diff-service.spec.js +57 -0
  142. package/dist/services/onboarding/templates/install-speed-kit-js-template.js +0 -5
  143. package/dist/services/onboarding/virtual-orestes-app/index.d.ts +3 -3
  144. package/dist/services/onboarding/virtual-orestes-app/index.js +5 -5
  145. package/dist/services/origin-request/index.d.ts +3 -0
  146. package/dist/services/origin-request/index.js +3 -0
  147. package/dist/services/origin-request/origin-fetcher.d.ts +52 -0
  148. package/dist/services/origin-request/origin-fetcher.js +93 -0
  149. package/dist/services/origin-request/origin-request-service-factory.js +19 -1
  150. package/dist/services/origin-request/origin-request-service.d.ts +6 -10
  151. package/dist/services/origin-request/origin-request-service.js +28 -70
  152. package/dist/services/origin-request/runtime-config-source.d.ts +36 -0
  153. package/dist/services/origin-request/runtime-config-source.js +59 -0
  154. package/dist/services/origin-request/spec/origin-error-cause.spec.d.ts +1 -0
  155. package/dist/services/{onboarding/spec/virtual-orestes-app → origin-request/spec}/origin-error-cause.spec.js +1 -1
  156. package/dist/services/origin-request/spec/origin-fetcher.spec.d.ts +1 -0
  157. package/dist/services/{onboarding/spec/virtual-orestes-app/crawler.spec.js → origin-request/spec/origin-fetcher.spec.js} +46 -15
  158. package/dist/services/prewarm/assets/asset-api-client.d.ts +6 -0
  159. package/dist/services/prewarm/assets/asset-api-client.js +6 -1
  160. package/dist/services/prewarm/pre-warm-service.d.ts +10 -0
  161. package/dist/services/prewarm/pre-warm-service.js +17 -0
  162. package/dist/services/prewarm/spec/pre-warm-service.spec.d.ts +1 -0
  163. package/dist/services/prewarm/spec/pre-warm-service.spec.js +72 -0
  164. package/dist/services/query-builder/query-builder-factory.js +3 -3
  165. package/dist/services/setup/os/tool-registry.d.ts +1 -1
  166. package/oclif.manifest.json +151 -19
  167. package/package.json +1 -2
  168. package/dist/commands/diff-url-parameters.d.ts +0 -19
  169. package/dist/commands/diff-url-parameters.js +0 -121
  170. package/dist/services/bundler/error/packjockel-error.d.ts +0 -3
  171. package/dist/services/bundler/error/packjockel-error.js +0 -3
  172. package/dist/services/bundler/packjockel-service.d.ts +0 -72
  173. package/dist/services/bundler/packjockel-service.js +0 -156
  174. package/dist/services/bundler/spec/packjockel-service.spec.js +0 -143
  175. package/dist/services/deploy/checks/post-deploy/deprecated-deployment-detection-check.d.ts +0 -15
  176. package/dist/services/deploy/checks/post-deploy/deprecated-deployment-detection-check.js +0 -70
  177. package/dist/services/deploy/error/check-module-file-error.d.ts +0 -4
  178. package/dist/services/deploy/error/check-module-file-error.js +0 -4
  179. package/dist/services/deploy/error/post-deploy-deployment-detection-error.d.ts +0 -4
  180. package/dist/services/deploy/error/post-deploy-deployment-detection-error.js +0 -4
  181. package/dist/services/onboarding/server-config/index.d.ts +0 -10
  182. package/dist/services/onboarding/server-config/index.js +0 -27
  183. package/dist/services/onboarding/virtual-orestes-app/crawler.d.ts +0 -31
  184. package/dist/services/onboarding/virtual-orestes-app/crawler.js +0 -98
  185. /package/dist/{services/bundler/spec/packjockel-service.spec.d.ts → helpers/spec/race.spec.d.ts} +0 -0
  186. /package/dist/services/{onboarding/spec/virtual-orestes-app/crawler.spec.d.ts → deploy/spec/deploy-service.spec.d.ts} +0 -0
  187. /package/dist/services/{onboarding/spec/virtual-orestes-app/origin-error-cause.spec.d.ts → deploy/spec/handler/install-resource-handler.spec.d.ts} +0 -0
  188. /package/dist/services/{onboarding/virtual-orestes-app → origin-request}/origin-error-cause.d.ts +0 -0
  189. /package/dist/services/{onboarding/virtual-orestes-app → origin-request}/origin-error-cause.js +0 -0
@@ -1,17 +1,16 @@
1
+ import { withoutNoise } from "../../diff/index.js";
2
+ import { DEFAULT_TYPE } from "../../origin-request/index.js";
1
3
  import { BaqendResponse } from "../browser/baqend-response.js";
2
4
  import { safe } from "../../../helpers/safe.js";
3
- import { originFetch } from "../../../helpers/origin-proxy.js";
4
- import { formatHtml } from "../../diff/index.js";
5
5
  export class RequestDiffService {
6
6
  diffService;
7
- documentHandlerRuntime;
8
- cli;
9
- constructor(diffService, documentHandlerRuntime, cli) {
7
+ pages;
8
+ constructor(diffService, pages) {
10
9
  this.diffService = diffService;
11
- this.documentHandlerRuntime = documentHandlerRuntime;
12
- this.cli = cli;
10
+ this.pages = pages;
13
11
  }
14
- async postDiff(requestOrigin, url, parametersString, autoCheck = false) {
12
+ async postDiff(requestOrigin, request) {
13
+ const { url, paramKeys: parametersString } = request;
15
14
  const parameters = parametersString
16
15
  .replace("[", "")
17
16
  .replace("]", "")
@@ -20,7 +19,7 @@ export class RequestDiffService {
20
19
  return parameter.trim();
21
20
  });
22
21
  const { urlWithParam, urlWithoutParam } = this.createUrlsForDiff(url, parameters);
23
- const result = await safe(this.diffLocal(urlWithParam, urlWithoutParam, autoCheck));
22
+ const result = await safe(this.diffLocal(urlWithParam, urlWithoutParam, request));
24
23
  if (result.success === true) {
25
24
  return this.createBaqendResponse(requestOrigin, JSON.stringify({
26
25
  status: "success",
@@ -32,47 +31,20 @@ export class RequestDiffService {
32
31
  error: result.error,
33
32
  }), "application/json; charset=UTF-8", [], 500);
34
33
  }
35
- async diffLocal(urlWithParameter, urlWithoutParameter, checkIsEqual = false) {
36
- const htmlWithParameter = await this.fetchHtml(urlWithParameter);
37
- const htmlWithoutParameter = await this.fetchHtml(urlWithoutParameter);
38
- const replaceRegEx = /<meta[^>]*name="baqend:asset_url"[^>]*>/;
39
- // <meta name="baqend:asset_url" content="https://www.kik.de/c/damen/damen-bekleidung-shirts-tops?page=7">
40
- const transformedHtmlWithParameterResponse = await this.documentHandlerRuntime.transform(
41
- // TODO: ma use original response content type
42
- "text/html; charset=utf-8", htmlWithParameter, "", urlWithParameter, {});
43
- if (Buffer.isBuffer(transformedHtmlWithParameterResponse.body)) {
44
- throw new TypeError("Can't diff body if it's a buffer");
34
+ /**
35
+ * Compares the page with and without the parameters, both fetched and
36
+ * rendered the way `sk diff` builds a side.
37
+ */
38
+ async diffLocal(urlWithParameter, urlWithoutParameter, { autoDiff, baseline }) {
39
+ const withParameter = await this.pages.fetch(urlWithParameter, DEFAULT_TYPE, "params", { baseline });
40
+ const withoutParameter = withoutNoise(withParameter, await this.pages.fetch(urlWithoutParameter, DEFAULT_TYPE, "pure", {
41
+ baseline,
42
+ }));
43
+ if (autoDiff) {
44
+ return this.diffService.isContentEqual(withParameter.content, withoutParameter.content);
45
45
  }
46
- let transformedHtmlWithParameter = transformedHtmlWithParameterResponse.body;
47
- transformedHtmlWithParameter = transformedHtmlWithParameter.replace(replaceRegEx, "");
48
- const transformedHtmlWithoutParameterResponse = await this.documentHandlerRuntime.transform(
49
- // TODO: ma use original response content type
50
- "text/html; charset=utf-8", htmlWithoutParameter, "", urlWithParameter, {});
51
- if (Buffer.isBuffer(transformedHtmlWithoutParameterResponse.body)) {
52
- throw new TypeError("Can't diff body if it's a buffer");
53
- }
54
- let transformedHtmlWithoutParameter = transformedHtmlWithoutParameterResponse.body;
55
- transformedHtmlWithoutParameter = transformedHtmlWithoutParameter.replace(replaceRegEx, "");
56
- if (checkIsEqual) {
57
- return this.diffService.isContentEqual(transformedHtmlWithParameter, transformedHtmlWithoutParameter);
58
- }
59
- const parametersFilePath = await this.diffService.writeContentToTemporalFile("params", formatHtml(transformedHtmlWithParameter));
60
- const pureFilePath = await this.diffService.writeContentToTemporalFile("pure", formatHtml(transformedHtmlWithoutParameter));
61
- return await this.diffService.executeDiff(parametersFilePath, pureFilePath);
62
- }
63
- async fetchHtml(url) {
64
- const response = await safe(originFetch(url, {
65
- headers: {
66
- "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36 SpeedKitCrawler/1.0",
67
- },
68
- },
69
- // The dashboard diff has always accepted a broken origin handshake here.
70
- { tolerateLegacyTls: true }));
71
- if (response.success !== true) {
72
- this.cli.writeError(`could not fetch "${url}": ${response.error}`);
73
- throw response.errorObj;
74
- }
75
- return response.data.text();
46
+ const { equal } = await this.diffService.compare(withParameter, withoutParameter);
47
+ return equal;
76
48
  }
77
49
  createUrlsForDiff(urlWithParameter, parameters) {
78
50
  const url = new URL(urlWithParameter);
@@ -76,7 +76,6 @@ export interface FetchRequestPausedEventHandler {
76
76
  }
77
77
  /** Where the Speed Kit parts come from when a customer brings none of its own. */
78
78
  export declare const DEFAULT_DF_PATH = "https://www.baqend.com/speed-kit/latest/dynamic-fetcher.js";
79
- export declare const DEFAULT_DOCUMENT_HANDLER_PATH = "https://www.baqend.com/speed-kit-handler/latest/DocumentHandler.js";
80
79
  export declare const DEFAULT_SNIPPET_PATH = "https://www.baqend.com/speed-kit/latest/snippet.js";
81
80
  export declare const DEFAULT_SW_PATH = "https://www.baqend.com/speed-kit/latest/sw.js";
82
81
  export interface SpeedKitServerConfigAuthentication {
@@ -206,7 +206,6 @@ export class BrowserContext {
206
206
  }
207
207
  /** Where the Speed Kit parts come from when a customer brings none of its own. */
208
208
  export const DEFAULT_DF_PATH = "https://www.baqend.com/speed-kit/latest/dynamic-fetcher.js";
209
- export const DEFAULT_DOCUMENT_HANDLER_PATH = "https://www.baqend.com/speed-kit-handler/latest/DocumentHandler.js";
210
209
  export const DEFAULT_SNIPPET_PATH = "https://www.baqend.com/speed-kit/latest/snippet.js";
211
210
  export const DEFAULT_SW_PATH = "https://www.baqend.com/speed-kit/latest/sw.js";
212
211
  export const USER_AGENT = {
@@ -61,7 +61,6 @@ export declare class OnboardingServiceFactory {
61
61
  private getAthenaClient;
62
62
  private getConfigApi;
63
63
  private getFetchEventHandler;
64
- private getSpeedKitServerConfig;
65
64
  private getIntegrationFiles;
66
65
  private sessionInfo;
67
66
  private getUrlForTest;
@@ -1,11 +1,12 @@
1
- import { DEFAULT_DF_PATH, DEFAULT_DOCUMENT_HANDLER_PATH, DEFAULT_SNIPPET_PATH, DEFAULT_SW_PATH, } from "./onboarding-model.js";
1
+ import { DEFAULT_DF_PATH, DEFAULT_SNIPPET_PATH, DEFAULT_SW_PATH, } from "./onboarding-model.js";
2
2
  import { safe } from "../../helpers/safe.js";
3
3
  import { INTEGRATION_FILES } from "../../models/files.js";
4
4
  import { AthenaServiceBuildContext, AthenaServiceFactory, } from "../athena/index.js";
5
5
  import { BundleServiceFactory } from "../bundler/index.js";
6
- import { DiffServiceBuildContext, DiffServiceFactory } from "../diff/index.js";
6
+ import { createDiffEngine, PageSource } from "../diff/index.js";
7
7
  import { DocumentHandlerRuntimeServiceBuildContext } from "../document-handler-runtime/document-handler-runtime-service-build-context.js";
8
8
  import DocumentHandlerRuntimeServiceFactory from "../document-handler-runtime/factory/document-handler-runtime-service-factory.js";
9
+ import { documentHandlerRuntimeFiles } from "../document-handler-runtime/runtime-files.js";
9
10
  import { CustomerConfig, ExternalFileReader, IntegrationApiServiceBuildContext, VirtualFile, ExternalRecipe, IntegrationApiFactory, } from "../integration-api/index.js";
10
11
  import { InstallFileRecipe } from "./config-renderer/install-file-recipe.js";
11
12
  import { SpeedKitInstallContext } from "./config-renderer/speed-kit-install-context.js";
@@ -41,7 +42,7 @@ import { DevtoolsExtensionApi } from "./browser/extension/devtools-extension-api
41
42
  import { ExtensionBridge } from "./browser/extension/extension-bridge.js";
42
43
  import { TodoCollector } from "./todo-collector.js";
43
44
  import { installBrowserOriginFetch } from "../browser-fetch/index.js";
44
- import { Crawler } from "./virtual-orestes-app/crawler.js";
45
+ import { OriginFetcher, RuntimeConfigSource } from "../origin-request/index.js";
45
46
  import { VirtualOrestesApp } from "./virtual-orestes-app/index.js";
46
47
  import { ConfigApiContext, ConfigApiServiceFactory, } from "../config-api/index.js";
47
48
  import * as fs from "node:fs";
@@ -311,47 +312,43 @@ export class OnboardingServiceFactory {
311
312
  }
312
313
  async getFetchEventHandler(files, customerConfig, documentHandler, cache, cli, browserContext, athenaClient, messageHandler, fileWatcher, speedKitServiceWorkerJs) {
313
314
  const configApi = this.getConfigApi(customerConfig.app);
314
- const serverConfig = await this.getSpeedKitServerConfig(configApi, cli);
315
- const crawler = new Crawler(cli, serverConfig);
315
+ const runtimeConfig = await new RuntimeConfigSource({
316
+ cli,
317
+ configApi,
318
+ }).load();
319
+ if (!runtimeConfig) {
320
+ // Everything the runtime config alone carries is gone for this session, including any
321
+ // override of a device variation. Those still resolve, from their built-in defaults only.
322
+ cli.writeWarning("without the runtime config, origin fetches carry no basic auth, no custom variation and no configured override of default, mobile or tablet — those fall back to the built-in defaults.");
323
+ }
324
+ const originFetcher = new OriginFetcher({ cli, config: runtimeConfig });
316
325
  const parameterQueryBuilder = await new QueryBuilderFactory(new ParameterQueryServiceBuildContext({
317
326
  environment: this.context.environment,
318
327
  configName: this.context.configName,
319
328
  customerPath: this.context.customerPath,
320
329
  isWindows: this.context.environment.oclif.windows,
321
330
  })).buildService();
322
- const DiffService = new DiffServiceFactory(new DiffServiceBuildContext({
323
- environment: this.context.environment,
324
- executablePath: this.context.environment.settings.diffExec,
325
- executableTemplate: this.context.environment.settings.diffExecTemplate,
326
- })).getService();
327
- const requestDiffService = new RequestDiffService(DiffService, documentHandler, cli);
331
+ const diffEngine = createDiffEngine(this.context.environment, "tool");
332
+ const pages = new PageSource({
333
+ fetcher: originFetcher,
334
+ documentHandler,
335
+ ignore: [],
336
+ });
337
+ const requestDiffService = new RequestDiffService(diffEngine, pages);
328
338
  const handlers = [
329
339
  new CustomerDomainDocumentResponse(customerConfig, new LocalDocumentRewrite(customerConfig, files.getByName(INTEGRATION_FILES.CONFIG.DYNAMIC_BLOCKS), files.getByName(INTEGRATION_FILES.CONFIG.DYNAMIC_STYLES), this.context.ignoreContentSecurityPolicy)),
330
340
  new SpeedKitInstallHtml(customerConfig, browserContext, new InstallSpeedKitHtmlTemplate()),
331
341
  new SpeedKitInstallJs(customerConfig, files.getByName(INTEGRATION_FILES.BUILD.INSTALL), cli),
332
342
  new CustomerServiceWorkerJs(customerConfig),
333
343
  speedKitServiceWorkerJs,
334
- new DashboardRequest(new Dashboard(customerConfig, parameterQueryBuilder, athenaClient, requestDiffService, new DiffAgainstCurrentPage(DiffService, documentHandler, crawler, cli), cache)),
344
+ new DashboardRequest(new Dashboard(customerConfig, parameterQueryBuilder, athenaClient, requestDiffService, new DiffAgainstCurrentPage(diffEngine, pages, cli), cache)),
335
345
  ];
336
- const orestesApp = new VirtualOrestesApp(customerConfig, crawler, documentHandler, cache, cli, messageHandler, this.context.ignoreContentSecurityPolicy);
346
+ const orestesApp = new VirtualOrestesApp(customerConfig, originFetcher, documentHandler, cache, cli, messageHandler, this.context.ignoreContentSecurityPolicy);
337
347
  // let config-file changes re-warm the dh-cache in the background
338
348
  fileWatcher.setCacheRefresher(orestesApp);
339
349
  handlers.push(new SpeedKitAssetRequest(customerConfig, orestesApp), new SpeedKitRumPiRequest(customerConfig, orestesApp));
340
350
  return new FetchEventHandler(handlers, customerConfig, cli, configApi);
341
351
  }
342
- async getSpeedKitServerConfig(configApi, cli) {
343
- cli.startAction("ONBOARDING:FETCH:RUNTIME_CONFIG", "get server config from app");
344
- const serverConfigResult = await safe(configApi.getServerConfig());
345
- if (serverConfigResult.success !== true) {
346
- cli.failAction("ONBOARDING:FETCH:RUNTIME_CONFIG");
347
- // Everything the runtime config alone carries is gone for this session, including any
348
- // override of a device variation. Those still resolve, from their built-in defaults only.
349
- cli.writeWarning("without the runtime config, origin fetches carry no basic auth, no custom variation and no configured override of default, mobile or tablet — those fall back to the built-in defaults. Are you logged in?");
350
- return;
351
- }
352
- cli.successAction("ONBOARDING:FETCH:RUNTIME_CONFIG");
353
- return JSON.parse(serverConfigResult.data);
354
- }
355
352
  async getIntegrationFiles(cli) {
356
353
  const projectRoot = path.resolve(this.context.customerPath, "../..");
357
354
  const speedKitPath = path.resolve(projectRoot, "node_modules", "speed-kit");
@@ -371,10 +368,7 @@ export class OnboardingServiceFactory {
371
368
  const integrationApi = new IntegrationApiFactory(integrationApiContext).buildService();
372
369
  const externalFileReader = new ExternalFileReader();
373
370
  const integrationApiArray = [
374
- {
375
- name: INTEGRATION_FILES.CUSTOM.DOCUMENT_HANDLER,
376
- recipe: new ExternalRecipe(DEFAULT_DOCUMENT_HANDLER_PATH, externalFileReader),
377
- },
371
+ ...documentHandlerRuntimeFiles(),
378
372
  ];
379
373
  // if speed-kit is npm-linked, use linked local files else fetch latest released sw and df
380
374
  if (isSpeedKitLinked) {
@@ -0,0 +1,57 @@
1
+ import { expect } from "chai";
2
+ import { describe, it } from "mocha";
3
+ import { DiffEngine, NoDiffSink } from "../../../diff/index.js";
4
+ import { RequestDiffService } from "../../dashboard/request-diff-service.js";
5
+ /** Answers each url with its page; a baseline marks `noise` as volatile. */
6
+ function pages(answers) {
7
+ const requests = [];
8
+ const source = {
9
+ requests,
10
+ async fetch(url, _variation, label, options) {
11
+ requests.push(`${url}${options?.baseline ? " +baseline" : ""}`);
12
+ return {
13
+ label,
14
+ content: answers[url],
15
+ baseline: options?.baseline
16
+ ? { lines: new Set(['<p id="0">']), reorders: false }
17
+ : undefined,
18
+ };
19
+ },
20
+ };
21
+ return source;
22
+ }
23
+ async function identical(service, baseline) {
24
+ const response = await service.postDiff("https://skonboarding.local", {
25
+ url: "https://shop.test/?a=1",
26
+ paramKeys: "[a]",
27
+ autoDiff: true,
28
+ baseline,
29
+ });
30
+ // CDP takes a fulfilled body base64-encoded.
31
+ return JSON.parse(Buffer.from(response.body, "base64").toString())
32
+ .isIdentical;
33
+ }
34
+ describe("RequestDiffService", () => {
35
+ const answers = {
36
+ "https://shop.test/?a=1": '<p id="1">',
37
+ "https://shop.test/": '<p id="2">',
38
+ };
39
+ it("counts a page's noise as a difference without a baseline", async () => {
40
+ const source = pages(answers);
41
+ const service = new RequestDiffService(new DiffEngine({ sink: new NoDiffSink() }), source);
42
+ expect(await identical(service)).to.equal(false);
43
+ expect(source.requests).to.deep.equal([
44
+ "https://shop.test/?a=1",
45
+ "https://shop.test/",
46
+ ]);
47
+ });
48
+ it("ignores a page's noise when the request asks for a baseline", async () => {
49
+ const source = pages(answers);
50
+ const service = new RequestDiffService(new DiffEngine({ sink: new NoDiffSink() }), source);
51
+ expect(await identical(service, true)).to.equal(true);
52
+ expect(source.requests).to.deep.equal([
53
+ "https://shop.test/?a=1 +baseline",
54
+ "https://shop.test/ +baseline",
55
+ ]);
56
+ });
57
+ });
@@ -5,11 +5,6 @@ export class InstallSpeedKitJsTemplate extends AbstractTemplate {
5
5
  // Config
6
6
  {{{config}}}
7
7
 
8
- {{#if configPlugins}}
9
- // The config writes SpeedKit.config, which packjockel exposes; the overrides below change it.
10
- var speedKit = SpeedKit.config;
11
- {{/if}}
12
-
13
8
  // Reset potential split test parameters
14
9
  speedKit.split = 1.0;
15
10
  speedKit.splitTestId = 'debug';
@@ -3,12 +3,12 @@ import { AbortResponse } from "../browser/abort-response.js";
3
3
  import { DocumentHandlerServer } from "../../document-handler-runtime/document-handler-server.js";
4
4
  import { Cache } from "../request-cache/cache.js";
5
5
  import { CliServiceInterface } from "../../cli/index.js";
6
- import { Crawler } from "./crawler.js";
6
+ import { OriginFetcher } from "../../origin-request/index.js";
7
7
  import { CustomerConfig } from "../../integration-api/index.js";
8
8
  import { DevtoolsExtensionApi } from "../browser/extension/devtools-extension-api.js";
9
9
  export declare class VirtualOrestesApp {
10
10
  private customerConfig;
11
- private crawler;
11
+ private originFetcher;
12
12
  private documentHandler;
13
13
  private cache;
14
14
  private cli;
@@ -16,7 +16,7 @@ export declare class VirtualOrestesApp {
16
16
  private removeContentSecurityPolicy;
17
17
  private localCacheErrors;
18
18
  private prerendererHintShown;
19
- constructor(customerConfig: CustomerConfig, crawler: Crawler, documentHandler: DocumentHandlerServer, cache: Cache, cli: CliServiceInterface, developmentToolsMessages: DevtoolsExtensionApi, removeContentSecurityPolicy?: boolean);
19
+ constructor(customerConfig: CustomerConfig, originFetcher: OriginFetcher, documentHandler: DocumentHandlerServer, cache: Cache, cli: CliServiceInterface, developmentToolsMessages: DevtoolsExtensionApi, removeContentSecurityPolicy?: boolean);
20
20
  fetchAssetResponse(event: Protocol.Fetch.RequestPausedEvent): Promise<Partial<Protocol.Fetch.FulfillRequestRequest> | AbortResponse>;
21
21
  /**
22
22
  * Re-warm all cached entries for the given origin url in the background.
@@ -1,6 +1,6 @@
1
1
  import { BaqendResponse } from "../browser/baqend-response.js";
2
2
  import { ABORT_SERVER_TIMING, ABORT_STATUS_CODE, isDocumentHandlerAbort, isLocalPrerendererUnreachable, LOCAL_PRERENDERER_PORT, prerenderFailure, } from "../../document-handler-runtime/document-handler-abort.js";
3
- import { isBlockedOriginStatus, ORIGIN_ERROR_STATUS_CODE, originErrorCauseName, originErrorCode, originErrorServerTiming, } from "./origin-error-cause.js";
3
+ import { isBlockedOriginStatus, ORIGIN_ERROR_STATUS_CODE, originErrorCauseName, originErrorCode, originErrorServerTiming, } from "../../origin-request/index.js";
4
4
  import { safe } from "../../../helpers/safe.js";
5
5
  import { META_CONTENT_TYPE_REGEX, } from "../onboarding-model.js";
6
6
  import iconv from "iconv-lite";
@@ -23,7 +23,7 @@ const SECURITY_HEADERS = [
23
23
  ];
24
24
  export class VirtualOrestesApp {
25
25
  customerConfig;
26
- crawler;
26
+ originFetcher;
27
27
  documentHandler;
28
28
  cache;
29
29
  cli;
@@ -36,9 +36,9 @@ export class VirtualOrestesApp {
36
36
  localCacheErrors = new Map();
37
37
  // The hint to start the pre-renderer is shown once; every later page gets one line.
38
38
  prerendererHintShown = false;
39
- constructor(customerConfig, crawler, documentHandler, cache, cli, developmentToolsMessages, removeContentSecurityPolicy = false) {
39
+ constructor(customerConfig, originFetcher, documentHandler, cache, cli, developmentToolsMessages, removeContentSecurityPolicy = false) {
40
40
  this.customerConfig = customerConfig;
41
- this.crawler = crawler;
41
+ this.originFetcher = originFetcher;
42
42
  this.documentHandler = documentHandler;
43
43
  this.cache = cache;
44
44
  this.cli = cli;
@@ -117,7 +117,7 @@ export class VirtualOrestesApp {
117
117
  const variation = UrlObject.searchParams.has("bqvariation")
118
118
  ? UrlObject.searchParams.get("bqvariation").trim()
119
119
  : "default";
120
- const fetchResult = await this.crawler.fetchRemote(originUrl, variation);
120
+ const fetchResult = await this.originFetcher.fetch(originUrl, variation);
121
121
  const response = fetchResult?.response;
122
122
  // URL we actually fetched (origin + variation query params); used for the redirect check below
123
123
  const fetchedUrl = fetchResult?.url;
@@ -2,3 +2,6 @@ export * from "./asset-variations.js";
2
2
  export * from "./origin-request-service-build-context.js";
3
3
  export * from "./origin-request-service-factory.js";
4
4
  export * from "./origin-request-service.js";
5
+ export * from "./origin-fetcher.js";
6
+ export * from "./origin-error-cause.js";
7
+ export * from "./runtime-config-source.js";
@@ -2,3 +2,6 @@ export * from "./asset-variations.js";
2
2
  export * from "./origin-request-service-build-context.js";
3
3
  export * from "./origin-request-service-factory.js";
4
4
  export * from "./origin-request-service.js";
5
+ export * from "./origin-fetcher.js";
6
+ export * from "./origin-error-cause.js";
7
+ export * from "./runtime-config-source.js";
@@ -0,0 +1,52 @@
1
+ import { CliServiceInterface } from "../cli/index.js";
2
+ import { ResolvedRequest } from "./asset-variations.js";
3
+ import { OriginRequestConfig } from "./runtime-config-source.js";
4
+ /** What a fetch reports through. */
5
+ export type OriginFetchLog = Pick<CliServiceInterface, "startAction" | "successAction" | "failAction" | "writeWarning" | "writeError">;
6
+ export interface OriginFetcherDeps {
7
+ readonly cli: OriginFetchLog;
8
+ /** The runtime config; absent when it could not be loaded. */
9
+ readonly config?: OriginRequestConfig;
10
+ /** A Host header for every request, as an origin-map host override sends it. */
11
+ readonly host?: string;
12
+ /**
13
+ * Refuse a variation the runtime config does not define, as the server does. Off, the request
14
+ * falls back to the base headers with a warning, so one bad variation does not stop a session.
15
+ */
16
+ readonly strict?: boolean;
17
+ }
18
+ /** A request to origin, built as Speed Kit's server builds it. */
19
+ export interface PreparedOriginRequest {
20
+ readonly url: string;
21
+ readonly init: RequestInit;
22
+ readonly resolved: ResolvedRequest;
23
+ }
24
+ /**
25
+ * Requests a page from origin exactly as the Baqend asset server does, so what a local tool sees
26
+ * matches what production would cache for the same variation. Onboarding's virtual app, the
27
+ * dashboard diffs, `sk origin-request` and `sk diff` all fetch through it.
28
+ *
29
+ * The resolution itself is `resolveRequest`: the base headers, the wildcard-then-variation order,
30
+ * the regex templates with their `${group}` substitution, and the query params. On top of it
31
+ * comes origin basic auth, which no variation carries.
32
+ */
33
+ export declare class OriginFetcher {
34
+ private readonly deps;
35
+ /** Warnings already shown, so a per-fetch condition is reported once per session. */
36
+ private readonly warned;
37
+ constructor(deps: OriginFetcherDeps);
38
+ /** The request, without sending it. */
39
+ prepare(originUrl: string, variation?: string): PreparedOriginRequest;
40
+ /**
41
+ * Fetches the page. Resolves to the response and the url actually requested — origin plus the
42
+ * variation's query params, which a caller compares against `response.url` to spot a redirect —
43
+ * or to undefined when the request failed; the error is reported.
44
+ */
45
+ fetch(originUrl: string, variation: string): Promise<{
46
+ response: Response;
47
+ url: string;
48
+ } | undefined>;
49
+ private resolve;
50
+ private authorization;
51
+ private warnOnce;
52
+ }
@@ -0,0 +1,93 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import ApplicationError from "../error-handling/error/application-error.js";
3
+ import { originFetch } from "../../helpers/origin-proxy.js";
4
+ import { resolveRequest, UndefinedVariationError, } from "./asset-variations.js";
5
+ /**
6
+ * Requests a page from origin exactly as the Baqend asset server does, so what a local tool sees
7
+ * matches what production would cache for the same variation. Onboarding's virtual app, the
8
+ * dashboard diffs, `sk origin-request` and `sk diff` all fetch through it.
9
+ *
10
+ * The resolution itself is `resolveRequest`: the base headers, the wildcard-then-variation order,
11
+ * the regex templates with their `${group}` substitution, and the query params. On top of it
12
+ * comes origin basic auth, which no variation carries.
13
+ */
14
+ export class OriginFetcher {
15
+ deps;
16
+ /** Warnings already shown, so a per-fetch condition is reported once per session. */
17
+ warned = new Set();
18
+ constructor(deps) {
19
+ this.deps = deps;
20
+ }
21
+ /** The request, without sending it. */
22
+ prepare(originUrl, variation) {
23
+ // A missing runtime config is not reported here; the load that failed says what it costs.
24
+ const variations = this.deps.config?.variations ?? {};
25
+ const resolved = this.resolve(variations, originUrl, variation);
26
+ const init = { headers: { ...resolved.headers } };
27
+ // Only a request that matched no specific variation falls back to origin basic auth. The
28
+ // wildcard applies to every request, so it does not count as a match.
29
+ const matchedSpecificVariation = resolved.appliedKeys.some((key) => key !== "*" && !key.startsWith("device-default:"));
30
+ if (!matchedSpecificVariation) {
31
+ const authentication = this.authorization(originUrl);
32
+ if (authentication)
33
+ init.headers["Authorization"] = authentication;
34
+ }
35
+ return { url: resolved.url, init, resolved };
36
+ }
37
+ /**
38
+ * Fetches the page. Resolves to the response and the url actually requested — origin plus the
39
+ * variation's query params, which a caller compares against `response.url` to spot a redirect —
40
+ * or to undefined when the request failed; the error is reported.
41
+ */
42
+ async fetch(originUrl, variation) {
43
+ const { url, init } = this.prepare(originUrl, variation);
44
+ const action = `ORIGIN:FETCH:${randomUUID()}`;
45
+ this.deps.cli.startAction(action, `[documentHandler:fetch]: ${url}`);
46
+ try {
47
+ const response = await originFetch(url, init);
48
+ this.deps.cli.successAction(action);
49
+ return { response, url };
50
+ }
51
+ catch (error) {
52
+ this.deps.cli.failAction(action, `error while fetching Asset: ${originUrl}`);
53
+ this.deps.cli.writeError(String(error));
54
+ return undefined;
55
+ }
56
+ }
57
+ resolve(variations, originUrl, variation) {
58
+ try {
59
+ return resolveRequest(variations, originUrl, variation, this.deps.host);
60
+ }
61
+ catch (error) {
62
+ // Only a custom variation reaches this branch: a device variation (default, mobile, tablet)
63
+ // never throws. It starts from its built-in default, and an entry for it in the runtime
64
+ // config is applied on top, so a configured user agent or header still wins.
65
+ if (!(error instanceof UndefinedVariationError))
66
+ throw error;
67
+ if (this.deps.strict)
68
+ throw new ApplicationError(error.message);
69
+ const defined = Object.keys(variations);
70
+ this.warnOnce(`undefined-variation:${variation}`, `variation ${variation} is not defined in the runtime config — ` +
71
+ (defined.length
72
+ ? `defined are: ${defined.join(", ")}. `
73
+ : "it defines no variations at all. ") +
74
+ "Falling back to the base headers.");
75
+ return resolveRequest({}, originUrl, undefined, this.deps.host);
76
+ }
77
+ }
78
+ authorization(originUrl) {
79
+ const origin = new URL(originUrl).origin;
80
+ for (const auth of this.deps.config?.authentications ?? []) {
81
+ if (auth.uri.includes(origin) && auth.type === "basic") {
82
+ return `Basic ${Buffer.from(`${auth.user}:${auth.password}`).toString("base64")}`;
83
+ }
84
+ }
85
+ return undefined;
86
+ }
87
+ warnOnce(key, message) {
88
+ if (this.warned.has(key))
89
+ return;
90
+ this.warned.add(key);
91
+ this.deps.cli.writeWarning(message);
92
+ }
93
+ }
@@ -2,6 +2,8 @@ import { ConfigApiContext, ConfigApiServiceFactory, } from "../config-api/index.
2
2
  import { IntegrationApiServiceBuildContext, IntegrationApiFactory, } from "../integration-api/index.js";
3
3
  import { applyOriginProxy } from "../../helpers/origin-proxy.js";
4
4
  import { OriginRequestService } from "./origin-request-service.js";
5
+ import { OriginFetcher } from "./origin-fetcher.js";
6
+ import { RuntimeConfigSource } from "./runtime-config-source.js";
5
7
  export class OriginRequestServiceFactory {
6
8
  context;
7
9
  service = undefined;
@@ -20,7 +22,23 @@ export class OriginRequestServiceFactory {
20
22
  const customer = files.getCustomerConfig().config;
21
23
  // `--json` owns stdout, so the proxy line stays out of a machine-readable run.
22
24
  applyOriginProxy(customer.chromeFlags, this.context.json ? undefined : cli);
23
- this.service = new OriginRequestService(this.context, cli, this.getConfigApi(customer.app));
25
+ const config = await new RuntimeConfigSource({
26
+ cli,
27
+ configApi: this.getConfigApi(customer.app),
28
+ fileFlag: "--variations",
29
+ }).load(this.context.variationsFile);
30
+ const usable = config && Object.keys(config.variations).length > 0;
31
+ if (config && !usable) {
32
+ cli.writeError("the config carries no speedKit.variations — nothing to simulate");
33
+ }
34
+ this.service = new OriginRequestService(this.context, cli, usable
35
+ ? new OriginFetcher({
36
+ cli,
37
+ config,
38
+ host: this.context.host,
39
+ strict: true,
40
+ })
41
+ : undefined);
24
42
  }
25
43
  async getIntegrationFiles() {
26
44
  const integrationApiContext = new IntegrationApiServiceBuildContext({
@@ -1,6 +1,6 @@
1
1
  import { CliServiceInterface } from "../cli/index.js";
2
- import { ConfigApiService } from "../config-api/index.js";
3
2
  import { OriginRequestOptions } from "./origin-request-service-build-context.js";
3
+ import { OriginFetcher } from "./origin-fetcher.js";
4
4
  /**
5
5
  * Fetches a page from origin with exactly the headers Speed Kit's server would send for a cache
6
6
  * variation, then reports what the origin answered.
@@ -19,15 +19,11 @@ import { OriginRequestOptions } from "./origin-request-service-build-context.js"
19
19
  export declare class OriginRequestService {
20
20
  private readonly context;
21
21
  private readonly cli;
22
- private readonly configApi;
23
- constructor(context: OriginRequestOptions, cli: CliServiceInterface, configApi: ConfigApiService);
22
+ /** Absent when the runtime config could not be used; the reason is reported. */
23
+ private readonly fetcher;
24
+ constructor(context: OriginRequestOptions, cli: CliServiceInterface,
25
+ /** Absent when the runtime config could not be used; the reason is reported. */
26
+ fetcher: OriginFetcher | undefined);
24
27
  run(): Promise<void>;
25
- /**
26
- * Reads `speedKit.variations` from the app's deployed runtime config, or from a local file when
27
- * `--variations` is given. Returns undefined when neither source is usable.
28
- */
29
- private loadVariations;
30
- private fetchRuntimeConfig;
31
- private readVariationsFile;
32
28
  private report;
33
29
  }