@platformos/platformos-check-docs-updater 0.0.2

package/dist/utils.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.memoize = exports.memo = exports.tap = exports.identity = exports.noop = void 0;
+const noop = () => { };
+exports.noop = noop;
+const identity = (x) => x;
+exports.identity = identity;
+const tap = (tappingFunction) => {
+    return (x) => {
+        tappingFunction(x);
+        return x;
+    };
+};
+exports.tap = tap;
+var platformos_check_common_1 = require("@platformos/platformos-check-common");
+Object.defineProperty(exports, "memo", { enumerable: true, get: function () { return platformos_check_common_1.memo; } });
+Object.defineProperty(exports, "memoize", { enumerable: true, get: function () { return platformos_check_common_1.memoize; } });
+//# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;;AACO,MAAM,IAAI,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;AAAhB,QAAA,IAAI,QAAY;AAEtB,MAAM,QAAQ,GAAG,CAAI,CAAI,EAAK,EAAE,CAAC,CAAC,CAAC;AAA7B,QAAA,QAAQ,YAAqB;AACnC,MAAM,GAAG,GAAG,CAAI,eAA+B,EAAE,EAAE;IACxD,OAAO,CAAC,CAAI,EAAE,EAAE;QACd,eAAe,CAAC,CAAC,CAAC,CAAC;QACnB,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;AACJ,CAAC,CAAC;AALW,QAAA,GAAG,OAKd;AAEF,+EAAoE;AAA3D,+GAAA,IAAI,OAAA;AAAE,kHAAA,OAAO,OAAA"}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@platformos/platformos-check-docs-updater",
+  "version": "0.0.2",
+  "description": "Scripts to initialize theme-check data with assets from the theme-liquid-docs repo.",
+  "main": "dist/index.js",
+  "author": "platformOS",
+  "license": "MIT",
+  "homepage": "https://github.com/Platform-OS/platformos-tools/tree/main/packages/platformos-check-docs-updater#readme",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Platform-OS/platformos-tools.git",
+    "directory": "packages/platformos-check-docs-updater"
+  },
+  "bugs": {
+    "url": "https://github.com/Platform-OS/platformos-tools/issues"
+  },
+  "publishConfig": {
+    "access": "public",
+    "@platformos:registry": "https://registry.npmjs.org"
+  },
+  "bin": {
+    "theme-docs": "./scripts/cli.js"
+  },
+  "scripts": {
+    "build": "yarn build:ts",
+    "build:ci": "yarn build",
+    "build:ts": "tsc -b tsconfig.build.json",
+    "postbuild": "node scripts/cli.js download data",
+    "test": "vitest",
+    "type-check": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@platformos/platformos-check-common": "0.0.2",
+    "env-paths": "^2.2.1",
+    "he": "^1.2.0",
+    "node-fetch": "^2.6.11"
+  },
+  "devDependencies": {
+    "@types/he": "^1.2.3",
+    "@types/node-fetch": "^2.6.4"
+  }
+}

package/scripts/cli.js

@@ -0,0 +1,48 @@
+#!/usr/bin/env node
+
+const path = require('path');
+const fs = require('fs');
+const { downloadThemeLiquidDocs, root } = require(path.resolve(__dirname, '../dist'));
+
+// Get the command line arguments
+const args = process.argv.slice(2);
+
+// Check if a command was provided
+if (args.length === 0) {
+  console.log(`
+Please provide a command.
+
+Usage:
+ download <dir> \t\tDownloads all docsets and JSON Schemas to the specified directory.
+ root \tPrints the default docsets root directory.
+ clear-cache \tClears the default docsets root directory.
+`);
+  process.exit(1);
+}
+
+// Handle the command
+switch (args[0]) {
+  case 'download':
+    if (args.length > 2) {
+      console.log('Please provide a directory to download docs into.');
+      process.exit(1);
+    }
+    console.log('Downloading docs...');
+
+    downloadThemeLiquidDocs(args[1], console.error.bind(console));
+
+    break;
+
+  case 'root':
+    console.log(root);
+    break;
+
+  case 'clear-cache':
+    console.log(`Removing '${root}'`);
+    fs.rmSync(root, { recursive: true });
+    break;
+
+  default:
+    console.log(`Unknown command: ${args[0]}`);
+    process.exit(1);
+}

package/src/index.ts

@@ -0,0 +1,8 @@
+export { ThemeLiquidDocsManager } from './themeLiquidDocsManager';
+export {
+  Resource,
+  Resources,
+  downloadResource,
+  downloadThemeLiquidDocs,
+  root,
+} from './themeLiquidDocsDownloader';

package/src/themeLiquidDocsDownloader.ts

@@ -0,0 +1,192 @@
+import { Mode } from '@platformos/platformos-check-common';
+import envPaths from 'env-paths';
+import fetch from 'node-fetch';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { Logger, noop, tap } from './utils';
+import he from 'he';
+const paths = envPaths('theme-liquid-docs');
+export const root = paths.cache;
+
+export const ThemeLiquidDocsRootFallback =
+  'https://raw.githubusercontent.com/Shopify/theme-liquid-docs/main';
+export const ThemeLiquidDocsRoot = 'https://documentation.platformos.com/api/liquid';
+export const ThemeCustomSchemas = ['tags', 'latest', 'objects', 'filters'];
+export const ThemeGraphQLSchema = 'https://documentation.platformos.com/api/graphql/schema';
+
+export type Resource = (typeof Resources)[number];
+export const Resources = [
+  'filters',
+  'objects',
+  'tags',
+  'shopify_system_translations',
+  'manifest_theme',
+  'manifest_theme_app_extension',
+] as const;
+
+export const Manifests = {
+  app: 'manifest_theme_app_extension',
+  theme: 'manifest_theme',
+} as const satisfies Record<Mode, Resource>;
+
+const THEME_LIQUID_DOCS: Record<Resource | 'latest', string> = {
+  filters: 'filters.json',
+  objects: 'objects.json',
+  tags: 'tags.json',
+  latest: 'latest.json',
+  shopify_system_translations: 'data/shopify_system_translations.json',
+  manifest_theme: 'schemas/manifest_theme.json',
+  manifest_theme_app_extension: 'schemas/manifest_theme_app_extension.json',
+};
+
+export async function downloadSchema(
+  relativeUri: string,
+  destination: string = root,
+  log: Logger = noop,
+) {
+  const remotePath = schemaUrl(relativeUri);
+  const localPath = schemaPath(relativeUri, destination);
+  const text = await download(remotePath, log);
+  await fs.writeFile(localPath, text, 'utf8');
+  return text;
+}
+
+export async function downloadResource(
+  resource: Resource | 'latest',
+  destination: string = root,
+  log: Logger = noop,
+) {
+  const remotePath = resourceUrl(resource);
+  const localPath = resourcePath(resource, destination);
+  const text = await download(remotePath, log);
+  await fs.writeFile(localPath, text, 'utf8');
+  return text;
+}
+
+export async function downloadGraphQLSchema(destination: string = root, log: Logger = noop) {
+  const localPath = graphQLPath(destination);
+  const text = await download(ThemeGraphQLSchema, log);
+  await fs.writeFile(localPath, he.decode(text), 'utf8');
+  return text;
+}
+
+export function graphQLPath(destination: string = root) {
+  return path.join(destination, `graphql.graphql`);
+}
+
+export async function download(path: string, log: Logger) {
+  if (path.startsWith('file:')) {
+    return await fs
+      .readFile(path.replace(/^file:/, ''), 'utf8')
+      .then(tap(() => log(`Using local file: ${path}`)))
+      .catch((error) => {
+        log(`Failed to read local file: ${path}`);
+        throw error;
+      });
+  } else {
+    log(path);
+    const res = await fetch(path);
+    return res.text();
+  }
+}
+
+export function resourcePath(resource: Resource | 'latest', destination: string = root) {
+  return path.join(destination, `${resource}.json`);
+}
+
+export function resourceUrl(resource: Resource | 'latest') {
+  let resourceRoot = `${ThemeLiquidDocsRootFallback}`;
+
+  if (ThemeCustomSchemas.includes(resource)) {
+    resourceRoot = ThemeLiquidDocsRoot;
+  }
+  const relativePath = THEME_LIQUID_DOCS[resource];
+  return `${resourceRoot}/${relativePath}`;
+}
+
+export function schemaPath(relativeUri: string, destination: string = root) {
+  return path.resolve(destination, path.basename(relativeUri));
+}
+
+export function schemaUrl(relativeUri: string) {
+  const schemaRoot = process.env.SHOPIFY_TLD_ROOT
+    ? `file:${process.env.SHOPIFY_TLD_ROOT}`
+    : ThemeLiquidDocsRootFallback;
+  return `${schemaRoot}/schemas/${relativeUri}`;
+}
+
+export async function exists(path: string) {
+  try {
+    await fs.stat(path);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+export async function downloadThemeLiquidDocs(destination: string, log: Logger) {
+  if (!(await exists(destination))) {
+    await fs.mkdir(destination);
+  }
+
+  const resources = ['latest'].concat(Resources) as (Resource | 'latest')[];
+  const resourceContents = await Promise.all(
+    resources.map((file) => {
+      return downloadResource(file, destination, log)
+        .then(
+          tap(() =>
+            log(
+              `Successfully downloaded latest resource:\n\t${resourceUrl(file)}\n\t> ${resourcePath(
+                file,
+                destination,
+              )}`,
+            ),
+          ),
+        )
+        .catch((error) => {
+          log(
+            `Failed to download latest resource:\n\t${resourceUrl(file)} to\n\t${resourcePath(
+              file,
+              destination,
+            )}\n${error}`,
+          );
+          throw error;
+        });
+    }),
+  );
+
+  const manifests = Object.values(Manifests)
+    .map((resource) => resources.indexOf(resource))
+    .map((index) => resourceContents[index])
+    .map((manifest) => JSON.parse(manifest));
+
+  const relativeUris = manifests.flatMap((manifest) =>
+    manifest.schemas.map((schema: { uri: string }) => schema.uri),
+  );
+
+  await Promise.all(
+    unique(relativeUris).map((uri) =>
+      downloadSchema(uri, destination, log)
+        .then(
+          tap(() =>
+            log(
+              `Successfully downloaded schema:\n\t${schemaUrl(uri)}\n\t> ${schemaPath(
+                uri,
+                destination,
+              )}`,
+            ),
+          ),
+        )
+        .catch((error) => {
+          log(`Failed to download schema: ${uri}, ${error}`);
+          throw error;
+        }),
+    ),
+  );
+
+  await downloadGraphQLSchema(destination, log);
+}
+
+function unique<T>(array: T[]): T[] {
+  return [...new Set(array).values()];
+}

package/src/themeLiquidDocsManager.spec.ts

@@ -0,0 +1,103 @@
+import { expect, describe, it, beforeEach, afterEach, vi } from 'vitest';
+import { ThemeLiquidDocsManager } from './themeLiquidDocsManager';
+import { downloadResource, Resources } from './themeLiquidDocsDownloader';
+import { noop } from './utils';
+
+vi.mock('./themeLiquidDocsDownloader', async (importOriginal) => {
+  const actual: any = await importOriginal();
+  return {
+    ...actual,
+    downloadResource: vi.fn(),
+  };
+});
+
+vi.mock('node:path', async () => {
+  return {
+    default: {
+      join: (...paths: string[]) => paths.join('/'),
+      resolve: () => '.',
+    },
+  };
+});
+
+vi.mock('env-paths', async () => {
+  return {
+    default: (appPath: string) => ({ cache: `MOCKED_CACHE/${appPath}` }),
+  };
+});
+
+vi.mock('node:fs/promises', async () => {
+  const fileSystem: Record<string, string> = {
+    'MOCKED_CACHE/theme-liquid-docs/filters.json': '[{"name": "upcase"}]',
+    'MOCKED_CACHE/theme-liquid-docs/objects.json': '[{"name": "product"}]',
+    'MOCKED_CACHE/theme-liquid-docs/tags.json': '[{"name": "if"}]',
+    'MOCKED_CACHE/theme-liquid-docs/latest.json': '{"revision": "1"}',
+    'MOCKED_CACHE/theme-liquid-docs/section_schema.json':
+      '{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"number"}},"required":["name","age"]}',
+    'MOCKED_CACHE/theme-liquid-docs/shopify_system_translations.json':
+      '{"shopify.checkout.general.cart": "Cart"}',
+  };
+
+  return {
+    default: {
+      readFile: vi.fn().mockImplementation((path) => fileSystem[path]),
+      mkdir: vi.fn(),
+    },
+  };
+});
+
+describe('Module: ThemeLiquidDocsManager', async () => {
+  let manager: ThemeLiquidDocsManager;
+
+  beforeEach(async () => {
+    manager = new ThemeLiquidDocsManager();
+  });
+
+  afterEach(async () => {
+    vi.clearAllMocks();
+  });
+
+  it('should not download remote files if the revision is stable', async () => {
+    await Promise.all([manager.filters(), manager.objects(), manager.tags()]);
+    expect(vi.mocked(downloadResource)).toHaveBeenNthCalledWith(
+      1,
+      'latest',
+      'MOCKED_CACHE/theme-liquid-docs',
+      noop,
+    );
+    expect(vi.mocked(downloadResource)).toHaveBeenCalledTimes(1);
+    for (const resource of Resources) {
+      expect(vi.mocked(downloadResource)).not.toHaveBeenCalledWith(resource, expect.any(String));
+    }
+  });
+
+  describe('Unit: filters', () => {
+    it('should return an array', async () => {
+      const filters = await manager.filters();
+      expect(filters).to.eql([{ name: 'upcase' }]);
+    });
+  });
+
+  describe('Unit: objects', () => {
+    it('should return an array', async () => {
+      const objects = await manager.objects();
+      expect(objects).to.eql([{ name: 'product' }]);
+    });
+  });
+
+  describe('Unit: tags', () => {
+    it('should return an array', async () => {
+      const tags = await manager.tags();
+      expect(tags).to.eql([{ name: 'if' }]);
+    });
+  });
+
+  describe('Unit: systemTranslations', () => {
+    it('should return the parsed JSON content of the system translations', async () => {
+      const systemTranslations = await manager.systemTranslations();
+      expect(systemTranslations).to.eql({
+        'shopify.checkout.general.cart': 'Cart',
+      });
+    });
+  });
+});

package/src/themeLiquidDocsManager.ts

@@ -0,0 +1,255 @@
+import {
+  FilterEntry,
+  JsonValidationSet,
+  Mode,
+  ObjectEntry,
+  SchemaDefinition,
+  TagEntry,
+  ThemeDocset,
+  Translations,
+} from '@platformos/platformos-check-common';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import {
+  Manifests,
+  Resource,
+  ThemeCustomSchemas,
+  ThemeLiquidDocsRoot,
+  ThemeLiquidDocsRootFallback,
+  downloadResource,
+  downloadThemeLiquidDocs,
+  exists,
+  graphQLPath,
+  resourcePath,
+  root,
+  schemaPath,
+} from './themeLiquidDocsDownloader';
+import { Logger, identity, memo, memoize, noop, tap } from './utils';
+
+type JSONSchemaManifest = { schemas: { uri: string; fileMatch?: string[] }[] };
+
+export class ThemeLiquidDocsManager implements ThemeDocset, JsonValidationSet {
+  constructor(private log: Logger = noop) {}
+
+  filters = memo(async (): Promise<FilterEntry[]> => {
+    return findSuitableResource(this.loaders('filters'), JSON.parse, [], this.log);
+  });
+
+  objects = memo(async (): Promise<ObjectEntry[]> => {
+    return findSuitableResource(this.loaders('objects'), JSON.parse, [], this.log);
+  });
+
+  liquidDrops = memo(async (): Promise<ObjectEntry[]> => {
+    return findSuitableResource(this.loaders('objects'), JSON.parse, [], this.log);
+  });
+
+  tags = memo(async (): Promise<TagEntry[]> => {
+    return findSuitableResource(this.loaders('tags'), JSON.parse, [], this.log);
+  });
+
+  graphQL = memo(async (): Promise<string | null> => {
+    return findSuitableResource(this.graphQLLoaders(), (x: string) => x, null, this.log);
+  });
+
+  systemTranslations = memo(async (): Promise<Translations> => {
+    return findSuitableResource(
+      this.loaders('shopify_system_translations'),
+      JSON.parse,
+      {},
+      this.log,
+    );
+  });
+
+  schemas = memoize(
+    (mode: Mode) =>
+      findSuitableResource<JSONSchemaManifest>(
+        this.loaders(Manifests[mode]),
+        JSON.parse,
+        {
+          schemas: [],
+        },
+        this.log,
+      ).then((manifest) => {
+        return Promise.all(
+          manifest.schemas.map(async (schemaDefinition): Promise<SchemaDefinition> => {
+            let schemaRoot = `${ThemeLiquidDocsRootFallback}/schemas`;
+
+            if (ThemeCustomSchemas.includes(schemaDefinition.uri)) {
+              schemaRoot = ThemeLiquidDocsRoot;
+            }
+            return {
+              uri: `${schemaRoot}/${schemaDefinition.uri}`,
+              fileMatch: schemaDefinition.fileMatch,
+              schema: await findSuitableResource(
+                this.schemaLoaders(schemaDefinition.uri),
+                identity,
+                '',
+                this.log,
+              ),
+            };
+          }),
+        );
+      }),
+    identity<Mode>,
+  );
+
+  /**
+   * The setup method checks that the latest revision matches the one from
+   * Shopify/theme-liquid-docs. If there's a diff in revision, it means
+   * that the documentations that you have locally are out of date.
+   *
+   * The setup method then downloads the other files.
+   */
+  setup = memo(async (): Promise<void> => {
+    try {
+      if (!(await exists(root))) {
+        await fs.mkdir(root, { recursive: true });
+      }
+
+      const local = await this.latestRevision();
+      await downloadResource('latest', root, this.log);
+      const remote = await this.latestRevision();
+      if (local !== remote) {
+        await downloadThemeLiquidDocs(root, this.log);
+      }
+    } catch (error) {
+      if (error instanceof Error) {
+        this.log(`Failed to setup with the following error: ${error.message}`);
+      }
+      return;
+    }
+  });
+
+  private async latestRevision(): Promise<string> {
+    const latest = await findSuitableResource(
+      [loader(() => this.load('latest'), 'loadLatestRevision')],
+      JSON.parse,
+      {},
+      this.log,
+    );
+    return latest['revision'] ?? '';
+  }
+
+  private async loadResource(name: Resource): Promise<string> {
+    // Always wait for setup first. Since it's memoized, this will always
+    // be the same promise.
+    await this.setup();
+    return this.load(name).then(tap(() => this.log(`Loaded resource from ${resourcePath(name)}`)));
+  }
+
+  private async load(name: Resource | 'latest') {
+    return fs.readFile(resourcePath(name), 'utf8');
+  }
+
+  private async loadSchema(relativeUri: string) {
+    return fs
+      .readFile(schemaPath(relativeUri), 'utf8')
+      .then(tap(() => this.log(`Loaded schema from ${schemaPath(relativeUri)}`)));
+  }
+
+  private async loadGraphQL() {
+    return fs.readFile(graphQLPath(), 'utf8').then(tap(() => this.log(`Loaded graphQL`)));
+  }
+
+  private loaders(name: Resource): Loader<string>[] {
+    return [
+      loader(() => this.loadResource(name), `loadResource(${name})`),
+      loader(() => fallbackResource(name, this.log), `fallbackResource(${name})`),
+    ];
+  }
+
+  private schemaLoaders(relativeUri: string): Loader<string>[] {
+    return [
+      loader(() => this.loadSchema(relativeUri), `loadSchema(${relativeUri})`),
+      loader(() => fallbackSchema(relativeUri, this.log), `fallbackSchema(${relativeUri})`),
+    ];
+  }
+
+  private graphQLLoaders(): Loader<string>[] {
+    return [
+      loader(() => this.loadGraphQL(), `loadGraphQL()`),
+      loader(() => fallbackGraphQL(this.log), `fallbackSchema()`),
+    ];
+  }
+}
+
+interface Loader<A> {
+  (): Promise<A>;
+  loaderName: string;
+}
+
+function loader<A>(fn: () => Promise<A>, loaderName: string): Loader<A> {
+  return Object.assign(fn, { loaderName });
+}
+
+/**
+ * Find the first resource that can be loaded and transformed.
+ *
+ * Will try to load the resource from the loaders in order. If the loader
+ * throws an error, it will try the next loader. If all loaders throw an
+ * error, it will return the default value.
+ *
+ * This should allow us to load the latest version of the resource if it's
+ * available, and fall back to the local version if it's not. If neither
+ * work, we'll just return the default value.
+ */
+async function findSuitableResource<B, A = string>(
+  dataLoaders: Loader<A>[],
+  transform: (x: A) => B,
+  defaultValue: B,
+  log: Logger,
+): Promise<B> {
+  for (const loader of dataLoaders) {
+    try {
+      return transform(await loader());
+    } catch (e) {
+      log(
+        `Failed to load or transform ${loader.loaderName} with the following error:\n${
+          e instanceof Error ? e.message : e
+        }`,
+      );
+
+      continue;
+    }
+  }
+  return defaultValue;
+}
+
+/**
+ * The root directory for the data files. This is different in the VS Code build
+ * (since those files are copied to the dist folder at a different relative path)
+ */
+function dataRoot() {
+  if (process.env.WEBPACK_MODE) {
+    return path.resolve(__dirname, './data');
+  } else {
+    return path.resolve(__dirname, '../data');
+  }
+}
+
+/** Returns the at-build-time path to the fallback data file. */
+async function fallbackResource(name: Resource, log: Logger): Promise<string> {
+  const sourcePath = path.resolve(dataRoot(), `${name}.json`);
+  return fs
+    .readFile(sourcePath, 'utf8')
+    .then(tap(() => log(`Loaded fallback resource\n\t${name} from\n\t${sourcePath}`)));
+}
+
+/** Returns the at-build-time path to the fallback schema file. */
+async function fallbackSchema(
+  /** e.g. themes/section.json */
+  relativeUri: string,
+  log: Logger,
+): Promise<string> {
+  const sourcePath = path.resolve(dataRoot(), path.basename(relativeUri));
+  return fs
+    .readFile(sourcePath, 'utf8')
+    .then(tap(() => log(`Loaded fallback schema\n\t${relativeUri} from\n\t${sourcePath}`)));
+}
+
+async function fallbackGraphQL(log: Logger): Promise<string> {
+  const sourcePath = path.resolve(dataRoot(), `graphql.graphql`);
+  return fs
+    .readFile(sourcePath, 'utf8')
+    .then(tap(() => log(`Loaded fallback graphQL from\n\t${sourcePath}`)));
+}

package/src/utils.ts

@@ -0,0 +1,12 @@
+export type Logger = (message: string) => void;
+export const noop = () => {};
+
+export const identity = <T>(x: T): T => x;
+export const tap = <T>(tappingFunction: (x: T) => void) => {
+  return (x: T) => {
+    tappingFunction(x);
+    return x;
+  };
+};
+
+export { memo, memoize } from '@platformos/platformos-check-common';

package/tsconfig.build.json

@@ -0,0 +1,7 @@
+{
+  "extends": "./tsconfig.json",
+  "exclude": ["**/*.spec.ts"],
+  "references": [
+    { "path": "../platformos-check-common/tsconfig.build.json" }
+  ]
+}