@sasjs/cli 6.29.0 → 6.30.1

package/build/commands/build/internal/mockSite/assets/sw.js

@@ -37,7 +37,6 @@
  */
 importScripts('host/runtime-core.js')
 importScripts('host/api-browser.js')
-importScripts('host/lint-browser.js')
 
 const R = self.SASjsMockRuntime
 
@@ -47,10 +46,6 @@ const R = self.SASjsMockRuntime
 // and the few capabilities a static origin does not have, which the package
 // reports as a 501 naming the missing service.
 const API = self.SASjsApi
-// The linter, bundled at build time from the installed @sasjs/api/code and
-// @sasjs/lint (see mockSite.ts). It is a separate script because the linter is
-// not part of the root package a service worker can import.
-const CODE = self.SASjsCode
 
 const DRIVE_STORE = R.driveStore
 
@@ -946,6 +941,110 @@ const homeService = API.createHomeService(async () => {
   }
 })
 
+/**
+ * The lint rules a SAS file is linted against: the nearest `.sasjslint` at or
+ * above its folder on the drive.
+ *
+ * This is the server's rule, applied here so the demo behaves like the
+ * installation it demonstrates: a folder in the tree carries rules for
+ * everything beneath it, and changing a rule is a plain edit of that file. It
+ * is why the build seeds a `.sasjslint` at the drive root - a file that is
+ * displayed but never read would be worse than absent.
+ *
+ * Paths go through the package's own normaliser, so a backslashed or escaping
+ * path resolves the way the drive routes resolve it rather than by a second set
+ * of string rules that could disagree with them.
+ *
+ * A file that is missing, is not JSON, or parses to something that is not an
+ * object is skipped and the walk continues upwards - a broken rules file must
+ * never break linting or formatting. A walk that reaches the root with nothing
+ * to show returns undefined, which lints against the linter's own defaults: the
+ * answer for a drive with no rules file at all.
+ */
+const LINT_CONFIG_FILE = '.sasjslint'
+
+/**
+ * The linter, bundled at build time from the installed @sasjs/api/code and
+ * @sasjs/lint (see mockSite.ts).
+ *
+ * Imported on the first code route rather than by the worker at boot: it is
+ * around 40KB of the worker's payload for two routes (`code/lint` and
+ * `code/format`), and the install has to finish before the page may make its
+ * first request - so every visit pays it, including the ones that never open the
+ * editor. The build stamps the version into the URL below, so the copy fetched
+ * here is cache-busted with the rest of the host.
+ */
+let codeService
+let linter
+
+const code = () => {
+  if (codeService) return codeService
+  importScripts('host/lint-browser.js')
+  linter = self.SASjsCode
+  codeService = linter.createCodeService({ lintConfig })
+  return codeService
+}
+
+/**
+ * The package's code service, resolved on first use.
+ *
+ * A proxy rather than an object listing `lint` and `format`: the package decides
+ * which methods a service has, and a facade would answer `undefined` for one a
+ * later version adds. Dispatch reads `services.code` once per request and only
+ * touches a method on a code route, so nothing below the first method access
+ * loads the linter.
+ */
+const lazyCode = () =>
+  new Proxy(
+    {},
+    {
+      get: (target, property) => {
+        // probing for a promise is not a method lookup: answering it must not
+        // pull 40KB
+        if (typeof property === 'symbol' || property === 'then')
+          return undefined
+        const service = code()
+        const value = service[property]
+        return typeof value === 'function' ? value.bind(service) : value
+      }
+    }
+  )
+
+const lintConfig = async (filePath) => {
+  // A path that escapes the drive has no folder to walk from, and a buffer that
+  // names no file has none either: both take the drive root's rules, which is
+  // what a server answers for each - a refused path there falls back to the
+  // root file rather than to the linter's defaults.
+  let folder = API.dirname(API.normaliseDrivePath(filePath) ?? '/')
+  for (;;) {
+    const configPath =
+      folder === '/' ? `/${LINT_CONFIG_FILE}` : `${folder}/${LINT_CONFIG_FILE}`
+    const contents = await DRIVE_STORE.read(configPath).catch((e) => {
+      // a piece that is still on the site is not a folder with no rules file:
+      // swallowing it here would lint against the defaults and never fetch it
+      if (e && e.code === R.PENDING_CODE) throw e
+      return null
+    })
+    const parsed = parseLintConfig(contents)
+    if (parsed) return new linter.LintConfig(parsed)
+    if (folder === '/') return undefined
+    folder = API.dirname(folder)
+  }
+}
+
+/** A `.sasjslint` as the object a configuration is built from, or null. */
+const parseLintConfig = (contents) => {
+  if (contents === null || contents === undefined) return null
+  const text =
+    typeof contents === 'string' ? contents : new TextDecoder().decode(contents)
+  try {
+    const parsed = JSON.parse(text)
+    return parsed && typeof parsed === 'object' ? parsed : null
+  } catch (e) {
+    return null
+  }
+}
+
 const SERVICES = {
   drive: API.createDriveService(DRIVE_STORE),
   // The package's browser fixture states one run time; this host serves two, and
@@ -970,10 +1069,11 @@ const SERVICES = {
   directory: API.createDirectoryService(IDENTITY_STORE, directorySource),
   appStream: APP_STREAM,
   home: homeService,
-  // the package's own code service over the bundled linter; the host passes no
-  // lint config, so a buffer is linted against the linter's defaults - a mock
-  // has no `.sasjslint` tree to resolve one from
-  code: CODE.createCodeService({})
+  // the package's own code service over the bundled linter, with the drive's
+  // own rules: a buffer is linted and formatted against the nearest
+  // `.sasjslint` at or above its folder, so editing that file changes the
+  // editor here exactly as it does on a server
+  code: lazyCode()
 }
 
 /** The routes this host performs itself, because the answer is not JSON. */
@@ -1302,6 +1402,15 @@ const subjectOf = async (request, query) => {
  * `_debug` means - is still the package's decision.
  */
 const runRoute = async (routeId, payload, subject) => {
+  // A program may read any file it likes - the drive is its filesystem - so
+  // every piece of a sharded drive is fetched here rather than guessed from the
+  // request. That is deliberately the whole drive and not just the app being
+  // run: replaying a program because a read failed would run its writes twice,
+  // so the safe choice costs the download of every other app's bundle the first
+  // time anything runs on a cold visit. A page that only lists apps never pays
+  // it; the saving this change is for is on that page.
+  await R.ensureAll()
+
   const query = payload.query
   const body = payload.body
   const tables = payload.files.map((file) => ({
@@ -1393,7 +1502,38 @@ const contentsResponse = (answer, path) => {
   })
 }
 
-const routeRequest = async (request, url) => {
+/**
+ * The shards a request names, fetched before it is answered.
+ *
+ * The drive of a sharded site is not in memory: the build splits a capture so a
+ * first page does not download every app's bundle, and a route that reads a path
+ * has to ask for the piece holding it. The two query parameters below are where
+ * a request names a path outright, which is the common case and the cheap one -
+ * whatever a route resolves from the drive itself is left to the retry in
+ * `routeRequest`, which knows the path from the read that failed rather than
+ * guessing at it from the URL.
+ */
+const ensureForRequest = async (url) => {
+  const paths = []
+  for (const name of ['_filePath', '_program']) {
+    const value = url.searchParams.get(name)
+    if (value) paths.push(value)
+  }
+  if (paths.length) await R.ensurePaths(paths)
+}
+
+/**
+ * One attempt at a request, with the body source a retry can copy again.
+ *
+ * A Request's body can only be read once, and a retry (see the catch below) is a
+ * second read of the same request - so the first attempt keeps an unread copy
+ * beside it and every attempt after that is made from a copy of the copy. Only a
+ * request with a body needs one, and `source` is what carries it through the
+ * recursion.
+ */
+const routeRequest = async (request, url, attempts = 0, source = null) => {
+  const hasBody = request.method !== 'GET' && request.method !== 'HEAD'
+  const replay = hasBody ? source || request.clone() : null
   const path = withoutBase(url.pathname).replace(/\/+$/, '') || '/'
   // `/api` is the short spelling of the API, and `/SASjsApi` the canonical one:
   // both are answered, so the alias is stripped once here and everything below -
@@ -1409,6 +1549,7 @@ const routeRequest = async (request, url) => {
     // (a static asset fetch raced by an API call) must not answer as nobody
     await ensureIdentity()
     await R.ensureReady()
+    await ensureForRequest(url)
 
     // The app stream is the server's own web surface rather than part of the
     // API: the listing, the landing page and the redirect from one to the other
@@ -1524,6 +1665,20 @@ const routeRequest = async (request, url) => {
       )
     return json(answer.body, answer.status)
   } catch (e) {
+    // A read of a file that is still on the site: the route reached a path the
+    // request did not name - an app asset, a `.sasjslint` above the edited file,
+    // a join a service resolves from the drive. The failure names the path, so
+    // the first attempt fetches exactly that piece; a second one in the same
+    // request means it wants more than one, so the rest of the drive comes in.
+    // Bounded, because a refusal that keeps repeating is a real one.
+    if (e && e.code === R.PENDING_CODE && attempts < 2) {
+      if (!attempts && e.path) await R.ensurePaths([e.path])
+      else await R.ensureAll()
+      // a fresh copy per attempt: the one this attempt read is spent
+      const next = replay ? replay.clone() : request
+      return await routeRequest(next, url, attempts + 1, replay)
+    }
+
     const message = (e && e.stack) || String(e)
     const status = (e && e.status) || 500
     const refusal = status >= 400 && status < 500
@@ -1628,7 +1783,7 @@ self.addEventListener('message', (event) => {
     case 'sasjs-load':
       event.waitUntil(
         bootIdentity(data)
-          .then(() => R.load(data.files || {}))
+          .then(() => R.load(data.files || {}, data.deleted || []))
           .then(async () => {
             await persistChanges()
             reply({
@@ -1791,7 +1946,22 @@ self.addEventListener('message', (event) => {
     }
 
     case 'sasjs-snapshot':
-      reply({ type: 'sasjs-snapshot', files: R.dumpAll() })
+      // a sharded drive is not all in memory, and the caller is exporting it -
+      // for persistence after a change, or as `mock-snapshot.json` for the next
+      // build. Whole means whole: the pieces nobody read are fetched first, or
+      // the export would quietly lose every app bundle it had not opened.
+      event.waitUntil(
+        R.ensureReady()
+          .then(() => R.ensureAll())
+          .then(() => reply({ type: 'sasjs-snapshot', files: R.dumpAll() }))
+          .catch((e) =>
+            reply({
+              type: 'sasjs-snapshot',
+              files: R.dumpAll(),
+              error: (e && e.message) || String(e)
+            })
+          )
+      )
       break
 
     case 'sasjs-stats':

package/build/commands/build/internal/mockSite/mockSite.d.ts

@@ -9,6 +9,11 @@ export interface MockSiteResult {
     webFileCount: number;
     /** True when a captured drive snapshot was shipped with the site. */
     snapshot: boolean;
+    /**
+     * How many pieces the captured drive ships in: the boot shard plus one per
+     * large folder or file. Zero when the site ships no snapshot.
+     */
+    snapshotShards: number;
     /** True when a recorded SAS session was shipped, for .sas runs to answer. */
     sasTranscript: boolean;
     /**
@@ -64,6 +69,24 @@ export declare const MAX_SEED_BYTES: number;
  * marker format changes.
  */
 export declare const DRIVE_FORMAT = 1;
+/**
+ * The size at which a captured file is fetched on its own rather than with its
+ * folder.
+ *
+ * A shard is a download the visitor waits for, and an app's folder holds both
+ * its bundles and the small files a listing shows - its logo, a config. Splitting
+ * on size keeps one from dragging the other: the tile asks for the folder's small
+ * files, the app's own page asks for the bundle.
+ */
+export declare const SHARD_FILE_BYTES: number;
+/**
+ * The size below which a folder's files ride in the boot shard.
+ *
+ * The boot shard is fetched before the first page renders, so it is paid by every
+ * visit. Folders under this size are cheaper there than as a request of their
+ * own; folders over it are only fetched when something reads them.
+ */
+export declare const SHARD_GROUP_BYTES: number;
 /**
  * Builds a static site that runs the project's JS services (its mocks) in the
  * browser, so a frontend can be tested with no server and no SAS.