@sasjs/cli 6.30.0 → 6.31.0

package/build/commands/build/internal/mockSite/assets/sw.js

@@ -37,7 +37,6 @@
  */
 importScripts('host/runtime-core.js')
 importScripts('host/api-browser.js')
-importScripts('host/lint-browser.js')
 
 const R = self.SASjsMockRuntime
 
@@ -47,10 +46,6 @@ const R = self.SASjsMockRuntime
 // and the few capabilities a static origin does not have, which the package
 // reports as a 501 naming the missing service.
 const API = self.SASjsApi
-// The linter, bundled at build time from the installed @sasjs/api/code and
-// @sasjs/lint (see mockSite.ts). It is a separate script because the linter is
-// not part of the root package a service worker can import.
-const CODE = self.SASjsCode
 
 const DRIVE_STORE = R.driveStore
 
@@ -968,6 +963,53 @@ const homeService = API.createHomeService(async () => {
  */
 const LINT_CONFIG_FILE = '.sasjslint'
 
+/**
+ * The linter, bundled at build time from the installed @sasjs/api/code and
+ * @sasjs/lint (see mockSite.ts).
+ *
+ * Imported on the first code route rather than by the worker at boot: it is
+ * around 40KB of the worker's payload for two routes (`code/lint` and
+ * `code/format`), and the install has to finish before the page may make its
+ * first request - so every visit pays it, including the ones that never open the
+ * editor. The build stamps the version into the URL below, so the copy fetched
+ * here is cache-busted with the rest of the host.
+ */
+let codeService
+let linter
+
+const code = () => {
+  if (codeService) return codeService
+  importScripts('host/lint-browser.js')
+  linter = self.SASjsCode
+  codeService = linter.createCodeService({ lintConfig })
+  return codeService
+}
+
+/**
+ * The package's code service, resolved on first use.
+ *
+ * A proxy rather than an object listing `lint` and `format`: the package decides
+ * which methods a service has, and a facade would answer `undefined` for one a
+ * later version adds. Dispatch reads `services.code` once per request and only
+ * touches a method on a code route, so nothing below the first method access
+ * loads the linter.
+ */
+const lazyCode = () =>
+  new Proxy(
+    {},
+    {
+      get: (target, property) => {
+        // probing for a promise is not a method lookup: answering it must not
+        // pull 40KB
+        if (typeof property === 'symbol' || property === 'then')
+          return undefined
+        const service = code()
+        const value = service[property]
+        return typeof value === 'function' ? value.bind(service) : value
+      }
+    }
+  )
+
 const lintConfig = async (filePath) => {
   // A path that escapes the drive has no folder to walk from, and a buffer that
   // names no file has none either: both take the drive root's rules, which is
@@ -977,9 +1019,14 @@ const lintConfig = async (filePath) => {
   for (;;) {
     const configPath =
       folder === '/' ? `/${LINT_CONFIG_FILE}` : `${folder}/${LINT_CONFIG_FILE}`
-    const contents = await DRIVE_STORE.read(configPath).catch(() => null)
+    const contents = await DRIVE_STORE.read(configPath).catch((e) => {
+      // a piece that is still on the site is not a folder with no rules file:
+      // swallowing it here would lint against the defaults and never fetch it
+      if (e && e.code === R.PENDING_CODE) throw e
+      return null
+    })
     const parsed = parseLintConfig(contents)
-    if (parsed) return new CODE.LintConfig(parsed)
+    if (parsed) return new linter.LintConfig(parsed)
     if (folder === '/') return undefined
     folder = API.dirname(folder)
   }
@@ -1026,7 +1073,7 @@ const SERVICES = {
   // own rules: a buffer is linted and formatted against the nearest
   // `.sasjslint` at or above its folder, so editing that file changes the
   // editor here exactly as it does on a server
-  code: CODE.createCodeService({ lintConfig })
+  code: lazyCode()
 }
 
 /** The routes this host performs itself, because the answer is not JSON. */
@@ -1355,6 +1402,15 @@ const subjectOf = async (request, query) => {
  * `_debug` means - is still the package's decision.
  */
 const runRoute = async (routeId, payload, subject) => {
+  // A program may read any file it likes - the drive is its filesystem - so
+  // every piece of a sharded drive is fetched here rather than guessed from the
+  // request. That is deliberately the whole drive and not just the app being
+  // run: replaying a program because a read failed would run its writes twice,
+  // so the safe choice costs the download of every other app's bundle the first
+  // time anything runs on a cold visit. A page that only lists apps never pays
+  // it; the saving this change is for is on that page.
+  await R.ensureAll()
+
   const query = payload.query
   const body = payload.body
   const tables = payload.files.map((file) => ({
@@ -1446,7 +1502,38 @@ const contentsResponse = (answer, path) => {
   })
 }
 
-const routeRequest = async (request, url) => {
+/**
+ * The shards a request names, fetched before it is answered.
+ *
+ * The drive of a sharded site is not in memory: the build splits a capture so a
+ * first page does not download every app's bundle, and a route that reads a path
+ * has to ask for the piece holding it. The two query parameters below are where
+ * a request names a path outright, which is the common case and the cheap one -
+ * whatever a route resolves from the drive itself is left to the retry in
+ * `routeRequest`, which knows the path from the read that failed rather than
+ * guessing at it from the URL.
+ */
+const ensureForRequest = async (url) => {
+  const paths = []
+  for (const name of ['_filePath', '_program']) {
+    const value = url.searchParams.get(name)
+    if (value) paths.push(value)
+  }
+  if (paths.length) await R.ensurePaths(paths)
+}
+
+/**
+ * One attempt at a request, with the body source a retry can copy again.
+ *
+ * A Request's body can only be read once, and a retry (see the catch below) is a
+ * second read of the same request - so the first attempt keeps an unread copy
+ * beside it and every attempt after that is made from a copy of the copy. Only a
+ * request with a body needs one, and `source` is what carries it through the
+ * recursion.
+ */
+const routeRequest = async (request, url, attempts = 0, source = null) => {
+  const hasBody = request.method !== 'GET' && request.method !== 'HEAD'
+  const replay = hasBody ? source || request.clone() : null
   const path = withoutBase(url.pathname).replace(/\/+$/, '') || '/'
   // `/api` is the short spelling of the API, and `/SASjsApi` the canonical one:
   // both are answered, so the alias is stripped once here and everything below -
@@ -1462,6 +1549,7 @@ const routeRequest = async (request, url) => {
     // (a static asset fetch raced by an API call) must not answer as nobody
     await ensureIdentity()
     await R.ensureReady()
+    await ensureForRequest(url)
 
     // The app stream is the server's own web surface rather than part of the
     // API: the listing, the landing page and the redirect from one to the other
@@ -1577,6 +1665,20 @@ const routeRequest = async (request, url) => {
       )
     return json(answer.body, answer.status)
   } catch (e) {
+    // A read of a file that is still on the site: the route reached a path the
+    // request did not name - an app asset, a `.sasjslint` above the edited file,
+    // a join a service resolves from the drive. The failure names the path, so
+    // the first attempt fetches exactly that piece; a second one in the same
+    // request means it wants more than one, so the rest of the drive comes in.
+    // Bounded, because a refusal that keeps repeating is a real one.
+    if (e && e.code === R.PENDING_CODE && attempts < 2) {
+      if (!attempts && e.path) await R.ensurePaths([e.path])
+      else await R.ensureAll()
+      // a fresh copy per attempt: the one this attempt read is spent
+      const next = replay ? replay.clone() : request
+      return await routeRequest(next, url, attempts + 1, replay)
+    }
+
     const message = (e && e.stack) || String(e)
     const status = (e && e.status) || 500
     const refusal = status >= 400 && status < 500
@@ -1630,6 +1732,22 @@ const handle = async (request, url) => {
   return response
 }
 
+// The console's own files under `/api/`, replaced at build time from the list the
+// builder writes the folder from, so the two cannot drift apart.
+const API_CONSOLE_FILES = [
+  'index.html',
+  'console.js',
+  'console.css',
+  'operations.json'
+]
+
+const CONSOLE_FILE = new RegExp(
+  '^/api/(' +
+    API_CONSOLE_FILES.map((name) => name.replace(/\./g, '\\.')).join('|') +
+    ')$',
+  'i'
+)
+
 self.addEventListener('fetch', (event) => {
   const url = new URL(event.request.url)
   if (url.origin !== self.location.origin) return
@@ -1662,12 +1780,20 @@ self.addEventListener('fetch', (event) => {
   // listing is the server's web route, and an interface that lists apps asks for
   // it on every visit.
   //
-  // `/api` and `/api/` on their own are the API's documentation page - a file
-  // the build copies into the site - so the worker stays out of the way there
-  // and the static host answers it. Every other `/api/...` is an API call on the
-  // short spelling, which routeRequest canonicalises to `/SASjsApi`.
+  // `/api` and `/api/` on their own are the console page - `api/index.html`,
+  // which this build writes - so the worker stays out of the way there and the
+  // static host answers it. Every other `/api/...` is an API call on the short
+  // spelling, which routeRequest canonicalises to `/SASjsApi`.
   if (/^\/api\/?$/i.test(path)) return
 
+  // And the console's own files, which live under that prefix: a path naming one
+  // of them is the page's, not an API call, so the static host answers it. Named
+  // one by one on purpose - an extension test would also pass through
+  // `/api/admin/logs/foo.log`, which is a real call on the short spelling. The
+  // list is replaced at build time with the one the builder writes the folder
+  // from, so a file added there cannot arrive here as an API path.
+  if (CONSOLE_FILE.test(path)) return
+
   if (!/^\/(SASjsApi|SASLogon|AppStream|api|app)(\/|$)/i.test(path)) return
   event.respondWith(handle(event.request, url))
 })
@@ -1681,7 +1807,7 @@ self.addEventListener('message', (event) => {
     case 'sasjs-load':
       event.waitUntil(
         bootIdentity(data)
-          .then(() => R.load(data.files || {}))
+          .then(() => R.load(data.files || {}, data.deleted || []))
           .then(async () => {
             await persistChanges()
             reply({
@@ -1844,7 +1970,22 @@ self.addEventListener('message', (event) => {
     }
 
     case 'sasjs-snapshot':
-      reply({ type: 'sasjs-snapshot', files: R.dumpAll() })
+      // a sharded drive is not all in memory, and the caller is exporting it -
+      // for persistence after a change, or as `mock-snapshot.json` for the next
+      // build. Whole means whole: the pieces nobody read are fetched first, or
+      // the export would quietly lose every app bundle it had not opened.
+      event.waitUntil(
+        R.ensureReady()
+          .then(() => R.ensureAll())
+          .then(() => reply({ type: 'sasjs-snapshot', files: R.dumpAll() }))
+          .catch((e) =>
+            reply({
+              type: 'sasjs-snapshot',
+              files: R.dumpAll(),
+              error: (e && e.message) || String(e)
+            })
+          )
+      )
       break
 
     case 'sasjs-stats':

package/build/commands/build/internal/mockSite/mockSite.d.ts

@@ -9,6 +9,13 @@ export interface MockSiteResult {
     webFileCount: number;
     /** True when a captured drive snapshot was shipped with the site. */
     snapshot: boolean;
+    /**
+     * How many pieces the captured drive ships in: the boot shard plus one per
+     * large folder or file. Zero when the site ships no snapshot.
+     */
+    snapshotShards: number;
+    /** True when the site carries the API console at <base>/api/. */
+    apiConsole: boolean;
     /** True when a recorded SAS session was shipped, for .sas runs to answer. */
     sasTranscript: boolean;
     /**
@@ -64,6 +71,24 @@ export declare const MAX_SEED_BYTES: number;
  * marker format changes.
  */
 export declare const DRIVE_FORMAT = 1;
+/**
+ * The size at which a captured file is fetched on its own rather than with its
+ * folder.
+ *
+ * A shard is a download the visitor waits for, and an app's folder holds both
+ * its bundles and the small files a listing shows - its logo, a config. Splitting
+ * on size keeps one from dragging the other: the tile asks for the folder's small
+ * files, the app's own page asks for the bundle.
+ */
+export declare const SHARD_FILE_BYTES: number;
+/**
+ * The size below which a folder's files ride in the boot shard.
+ *
+ * The boot shard is fetched before the first page renders, so it is paid by every
+ * visit. Folders under this size are cheaper there than as a request of their
+ * own; folders over it are only fetched when something reads them.
+ */
+export declare const SHARD_GROUP_BYTES: number;
 /**
  * Builds a static site that runs the project's JS services (its mocks) in the
  * browser, so a frontend can be tested with no server and no SAS.