decap-cms-lib-util 2.16.0-beta.0

package/src/__tests__/backendUtil.spec.js

@@ -0,0 +1,97 @@
+import { oneLine } from 'common-tags';
+import nock from 'nock';
+
+import { parseLinkHeader, getAllResponses, getPathDepth, filterByExtension } from '../backendUtil';
+
+describe('parseLinkHeader', () => {
+  it('should return the right rel urls', () => {
+    const url = 'https://api.github.com/resource';
+    const link = oneLine`
+      <${url}?page=1>; rel="first",
+      <${url}?page=2>; rel="prev",
+      <${url}?page=4>; rel="next",
+      <${url}?page=5>; rel="last"
+    `;
+    const linkHeader = parseLinkHeader(link);
+
+    expect(linkHeader.next).toBe(`${url}?page=4`);
+    expect(linkHeader.last).toBe(`${url}?page=5`);
+    expect(linkHeader.first).toBe(`${url}?page=1`);
+    expect(linkHeader.prev).toBe(`${url}?page=2`);
+  });
+});
+
+describe('getAllResponses', () => {
+  function generatePulls(length) {
+    return Array.from({ length }, (_, id) => {
+      return { id: id + 1, number: `134${id}`, state: 'open' };
+    });
+  }
+
+  function createLinkHeaders({ page, pageCount }) {
+    const pageNum = parseInt(page, 10);
+    const pageCountNum = parseInt(pageCount, 10);
+    const url = 'https://api.github.com/pulls';
+
+    function link(linkPage) {
+      return `<${url}?page=${linkPage}>`;
+    }
+
+    const linkHeader = oneLine`
+      ${pageNum === 1 ? '' : `${link(1)}; rel="first",`}
+      ${pageNum === pageCountNum ? '' : `${link(pageCount)}; rel="last",`}
+      ${pageNum === 1 ? '' : `${link(pageNum - 1)}; rel="prev",`}
+      ${pageNum === pageCountNum ? '' : `${link(pageNum + 1)}; rel="next",`}
+    `.slice(0, -1);
+
+    return { Link: linkHeader };
+  }
+
+  function interceptCall({ perPage = 30, repeat = 1, data = [] } = {}) {
+    nock('https://api.github.com')
+      .get('/pulls')
+      .query(true)
+      .times(repeat)
+      .reply(uri => {
+        const searchParams = new URLSearchParams(uri.split('?')[1]);
+        const page = searchParams.get('page') || 1;
+        const pageCount = data.length <= perPage ? 1 : Math.ceil(data.length / perPage);
+        const pageLastIndex = page * perPage;
+        const pageFirstIndex = pageLastIndex - perPage;
+        const resp = data.slice(pageFirstIndex, pageLastIndex);
+        return [200, resp, createLinkHeaders({ page, pageCount })];
+      });
+  }
+
+  it('should return all paged response', async () => {
+    interceptCall({ repeat: 3, data: generatePulls(70) });
+    const res = await getAllResponses('https://api.github.com/pulls', {}, 'next', url => url);
+    const pages = await Promise.all(res.map(res => res.json()));
+
+    expect(pages[0]).toHaveLength(30);
+    expect(pages[1]).toHaveLength(30);
+    expect(pages[2]).toHaveLength(10);
+  });
+});
+
+describe('getPathDepth', () => {
+  it('should return 1 for empty string', () => {
+    expect(getPathDepth('')).toBe(1);
+  });
+
+  it('should return 2 for path of one nested folder', () => {
+    expect(getPathDepth('{{year}}/{{slug}}')).toBe(2);
+  });
+});
+
+describe('filterByExtension', () => {
+  it('should return true when extension matches', () => {
+    expect(filterByExtension({ path: 'file.html.md' }, '.html.md')).toBe(true);
+    expect(filterByExtension({ path: 'file.html.md' }, 'html.md')).toBe(true);
+  });
+
+  it("should return false when extension doesn't match", () => {
+    expect(filterByExtension({ path: 'file.json' }, '.html.md')).toBe(false);
+    expect(filterByExtension({ path: 'file.json' }, 'html.md')).toBe(false);
+  });
+});

package/src/__tests__/implementation.spec.js

@@ -0,0 +1,58 @@
+import { getMediaAsBlob, getMediaDisplayURL } from '../implementation';
+
+describe('implementation', () => {
+  describe('getMediaAsBlob', () => {
+    it('should return response blob on non svg file', async () => {
+      const blob = {};
+      const readFile = jest.fn().mockResolvedValue(blob);
+
+      await expect(getMediaAsBlob('static/media/image.png', 'sha', readFile)).resolves.toBe(blob);
+
+      expect(readFile).toHaveBeenCalledTimes(1);
+      expect(readFile).toHaveBeenCalledWith('static/media/image.png', 'sha', {
+        parseText: false,
+      });
+    });
+
+    it('should return text blob on svg file', async () => {
+      const text = 'svg';
+      const readFile = jest.fn().mockResolvedValue(text);
+
+      await expect(getMediaAsBlob('static/media/logo.svg', 'sha', readFile)).resolves.toEqual(
+        new Blob([text], { type: 'image/svg+xml' }),
+      );
+
+      expect(readFile).toHaveBeenCalledTimes(1);
+      expect(readFile).toHaveBeenCalledWith('static/media/logo.svg', 'sha', {
+        parseText: true,
+      });
+    });
+  });
+
+  describe('getMediaDisplayURL', () => {
+    it('should return createObjectURL result', async () => {
+      const blob = {};
+      const readFile = jest.fn().mockResolvedValue(blob);
+      const semaphore = { take: jest.fn(callback => callback()), leave: jest.fn() };
+
+      global.URL.createObjectURL = jest
+        .fn()
+        .mockResolvedValue('blob:http://localhost:8080/blob-id');
+
+      await expect(
+        getMediaDisplayURL({ path: 'static/media/image.png', id: 'sha' }, readFile, semaphore),
+      ).resolves.toBe('blob:http://localhost:8080/blob-id');
+
+      expect(semaphore.take).toHaveBeenCalledTimes(1);
+      expect(semaphore.leave).toHaveBeenCalledTimes(1);
+
+      expect(readFile).toHaveBeenCalledTimes(1);
+      expect(readFile).toHaveBeenCalledWith('static/media/image.png', 'sha', {
+        parseText: false,
+      });
+
+      expect(global.URL.createObjectURL).toHaveBeenCalledTimes(1);
+      expect(global.URL.createObjectURL).toHaveBeenCalledWith(blob);
+    });
+  });
+});

package/src/__tests__/path.spec.js

@@ -0,0 +1,53 @@
+import { fileExtensionWithSeparator, fileExtension } from '../path';
+
+describe('fileExtensionWithSeparator', () => {
+  it('should return the extension of a file', () => {
+    expect(fileExtensionWithSeparator('index.html')).toEqual('.html');
+  });
+
+  it('should return the extension of a file path', () => {
+    expect(fileExtensionWithSeparator('/src/main/index.html')).toEqual('.html');
+  });
+
+  it('should return the extension of a file path with trailing slash', () => {
+    expect(fileExtensionWithSeparator('/src/main/index.html/')).toEqual('.html');
+  });
+
+  it('should return the extension for an extension with two ..', () => {
+    expect(fileExtensionWithSeparator('/src/main/index..html')).toEqual('.html');
+  });
+
+  it('should return an empty string for the parent path ..', () => {
+    expect(fileExtensionWithSeparator('..')).toEqual('');
+  });
+
+  it('should return an empty string if the file has no extension', () => {
+    expect(fileExtensionWithSeparator('/src/main/index')).toEqual('');
+  });
+});
+
+describe('fileExtension', () => {
+  it('should return the extension of a file', () => {
+    expect(fileExtension('index.html')).toEqual('html');
+  });
+
+  it('should return the extension of a file path', () => {
+    expect(fileExtension('/src/main/index.html')).toEqual('html');
+  });
+
+  it('should return the extension of a file path with trailing slash', () => {
+    expect(fileExtension('/src/main/index.html/')).toEqual('html');
+  });
+
+  it('should return the extension for an extension with two ..', () => {
+    expect(fileExtension('/src/main/index..html')).toEqual('html');
+  });
+
+  it('should return an empty string for the parent path ..', () => {
+    expect(fileExtension('..')).toEqual('');
+  });
+
+  it('should return an empty string if the file has no extension', () => {
+    expect(fileExtension('/src/main/index')).toEqual('');
+  });
+});

package/src/__tests__/unsentRequest.spec.js

@@ -0,0 +1,19 @@
+import unsentRequest from '../unsentRequest';
+
+describe('unsentRequest', () => {
+  describe('withHeaders', () => {
+    it('should create new request with headers', () => {
+      expect(unsentRequest.withHeaders({ Authorization: 'token' })('path').toJS()).toEqual({
+        url: 'path',
+        headers: { Authorization: 'token' },
+      });
+    });
+
+    it('should add headers to existing request', () => {
+      expect(unsentRequest.withHeaders({ Authorization: 'token' }, 'path').toJS()).toEqual({
+        url: 'path',
+        headers: { Authorization: 'token' },
+      });
+    });
+  });
+});

package/src/asyncLock.ts

@@ -0,0 +1,43 @@
+import semaphore from 'semaphore';
+
+export type AsyncLock = { release: () => void; acquire: () => Promise<boolean> };
+
+export function asyncLock(): AsyncLock {
+  let lock = semaphore(1);
+
+  function acquire(timeout = 15000) {
+    const promise = new Promise<boolean>(resolve => {
+      // this makes sure a caller doesn't gets stuck forever awaiting on the lock
+      const timeoutId = setTimeout(() => {
+        // we reset the lock in that case to allow future consumers to use it without being blocked
+        lock = semaphore(1);
+        resolve(false);
+      }, timeout);
+
+      lock.take(() => {
+        clearTimeout(timeoutId);
+        resolve(true);
+      });
+    });
+
+    return promise;
+  }
+
+  function release() {
+    try {
+      // suppress too many calls to leave error
+      lock.leave();
+    } catch (e) {
+      // calling 'leave' too many times might not be good behavior
+      // but there is no reason to completely fail on it
+      if (e.message !== 'leave called too many times.') {
+        throw e;
+      } else {
+        console.warn('leave called too many times.');
+        lock = semaphore(1);
+      }
+    }
+  }
+
+  return { acquire, release };
+}

package/src/backendUtil.ts

@@ -0,0 +1,120 @@
+import { flow, fromPairs } from 'lodash';
+import { map } from 'lodash/fp';
+import { fromJS } from 'immutable';
+
+import unsentRequest from './unsentRequest';
+import APIError from './APIError';
+
+type Formatter = (res: Response) => Promise<string | Blob | unknown>;
+
+export function filterByExtension(file: { path: string }, extension: string) {
+  const path = file?.path || '';
+  return path.endsWith(extension.startsWith('.') ? extension : `.${extension}`);
+}
+
+function catchFormatErrors(format: string, formatter: Formatter) {
+  return (res: Response) => {
+    try {
+      return formatter(res);
+    } catch (err) {
+      throw new Error(
+        `Response cannot be parsed into the expected format (${format}): ${err.message}`,
+      );
+    }
+  };
+}
+
+const responseFormatters = fromJS({
+  json: async (res: Response) => {
+    const contentType = res.headers.get('Content-Type') || '';
+    if (!contentType.startsWith('application/json') && !contentType.startsWith('text/json')) {
+      throw new Error(`${contentType} is not a valid JSON Content-Type`);
+    }
+    return res.json();
+  },
+  text: async (res: Response) => res.text(),
+  blob: async (res: Response) => res.blob(),
+}).mapEntries(([format, formatter]: [string, Formatter]) => [
+  format,
+  catchFormatErrors(format, formatter),
+]);
+
+export async function parseResponse(
+  res: Response,
+  { expectingOk = true, format = 'text', apiName = '' },
+) {
+  let body;
+  try {
+    const formatter = responseFormatters.get(format, false);
+    if (!formatter) {
+      throw new Error(`${format} is not a supported response format.`);
+    }
+    body = await formatter(res);
+  } catch (err) {
+    throw new APIError(err.message, res.status, apiName);
+  }
+  if (expectingOk && !res.ok) {
+    const isJSON = format === 'json';
+    const message = isJSON ? body.message || body.msg || body.error?.message : body;
+    throw new APIError(isJSON && message ? message : body, res.status, apiName);
+  }
+  return body;
+}
+
+export function responseParser(options: {
+  expectingOk?: boolean;
+  format: string;
+  apiName: string;
+}) {
+  return (res: Response) => parseResponse(res, options);
+}
+
+export function parseLinkHeader(header: string | null) {
+  if (!header) {
+    return {};
+  }
+  return flow([
+    linksString => linksString.split(','),
+    map((str: string) => str.trim().split(';')),
+    map(([linkStr, keyStr]) => [
+      keyStr.match(/rel="(.*?)"/)[1],
+      linkStr
+        .trim()
+        .match(/<(.*?)>/)[1]
+        .replace(/\+/g, '%20'),
+    ]),
+    fromPairs,
+  ])(header);
+}
+
+export async function getAllResponses(
+  url: string,
+  options: { headers?: {} } = {},
+  linkHeaderRelName: string,
+  nextUrlProcessor: (url: string) => string,
+) {
+  const maxResponses = 30;
+  let responseCount = 1;
+
+  let req = unsentRequest.fromFetchArguments(url, options);
+
+  const pageResponses = [];
+
+  while (req && responseCount < maxResponses) {
+    const pageResponse = await unsentRequest.performRequest(req);
+    const linkHeader = pageResponse.headers.get('Link');
+    const nextURL = linkHeader && parseLinkHeader(linkHeader)[linkHeaderRelName];
+
+    const { headers = {} } = options;
+    req = nextURL && unsentRequest.fromFetchArguments(nextUrlProcessor(nextURL), { headers });
+    pageResponses.push(pageResponse);
+    responseCount++;
+  }
+
+  return pageResponses;
+}
+
+export function getPathDepth(path: string) {
+  const depth = path.split('/').length;
+  return depth;
+}

package/src/getBlobSHA.ts

@@ -0,0 +1,12 @@
+import { sha256 } from 'js-sha256';
+
+export default (blob: Blob): Promise<string> =>
+  new Promise((resolve, reject) => {
+    const fr = new FileReader();
+    fr.onload = ({ target }) => resolve(sha256(target?.result || ''));
+    fr.onerror = err => {
+      fr.abort();
+      reject(err);
+    };
+    fr.readAsArrayBuffer(blob);
+  });

package/src/git-lfs.ts

@@ -0,0 +1,133 @@
+//
+// Pointer file parsing
+
+import { filter, flow, fromPairs, map } from 'lodash/fp';
+
+import getBlobSHA from './getBlobSHA';
+
+import type { AssetProxy } from './implementation';
+
+export interface PointerFile {
+  size: number;
+  sha: string;
+}
+
+function splitIntoLines(str: string) {
+  return str.split('\n');
+}
+
+function splitIntoWords(str: string) {
+  return str.split(/\s+/g);
+}
+
+function isNonEmptyString(str: string) {
+  return str !== '';
+}
+
+const withoutEmptyLines = flow([map((str: string) => str.trim()), filter(isNonEmptyString)]);
+export const parsePointerFile: (data: string) => PointerFile = flow([
+  splitIntoLines,
+  withoutEmptyLines,
+  map(splitIntoWords),
+  fromPairs,
+  ({ size, oid, ...rest }) => ({
+    size: parseInt(size),
+    sha: oid?.split(':')[1],
+    ...rest,
+  }),
+]);
+
+//
+// .gitattributes file parsing
+
+function removeGitAttributesCommentsFromLine(line: string) {
+  return line.split('#')[0];
+}
+
+function parseGitPatternAttribute(attributeString: string) {
+  // There are three kinds of attribute settings:
+  // - a key=val pair sets an attribute to a specific value
+  // - a key without a value and a leading hyphen sets an attribute to false
+  // - a key without a value and no leading hyphen sets an attribute
+  //   to true
+  if (attributeString.includes('=')) {
+    return attributeString.split('=');
+  }
+  if (attributeString.startsWith('-')) {
+    return [attributeString.slice(1), false];
+  }
+  return [attributeString, true];
+}
+
+const parseGitPatternAttributes = flow([map(parseGitPatternAttribute), fromPairs]);
+
+const parseGitAttributesPatternLine = flow([
+  splitIntoWords,
+  ([pattern, ...attributes]) => [pattern, parseGitPatternAttributes(attributes)],
+]);
+
+const parseGitAttributesFileToPatternAttributePairs = flow([
+  splitIntoLines,
+  map(removeGitAttributesCommentsFromLine),
+  withoutEmptyLines,
+  map(parseGitAttributesPatternLine),
+]);
+
+export const getLargeMediaPatternsFromGitAttributesFile = flow([
+  parseGitAttributesFileToPatternAttributePairs,
+  filter(
+    ([, attributes]) =>
+      attributes.filter === 'lfs' && attributes.diff === 'lfs' && attributes.merge === 'lfs',
+  ),
+  map(([pattern]) => pattern),
+]);
+
+export function createPointerFile({ size, sha }: PointerFile) {
+  return `\
+version https://git-lfs.github.com/spec/v1
+oid sha256:${sha}
+size ${size}
+`;
+}
+
+export async function getPointerFileForMediaFileObj(
+  client: { uploadResource: (pointer: PointerFile, resource: Blob) => Promise<string> },
+  fileObj: File,
+  path: string,
+) {
+  const { name, size } = fileObj;
+  const sha = await getBlobSHA(fileObj);
+  await client.uploadResource({ sha, size }, fileObj);
+  const pointerFileString = createPointerFile({ sha, size });
+  const pointerFileBlob = new Blob([pointerFileString]);
+  const pointerFile = new File([pointerFileBlob], name, { type: 'text/plain' });
+  const pointerFileSHA = await getBlobSHA(pointerFile);
+  return {
+    fileObj: pointerFile,
+    size: pointerFileBlob.size,
+    sha: pointerFileSHA,
+    raw: pointerFileString,
+    path,
+  };
+}
+
+export async function getLargeMediaFilteredMediaFiles(
+  client: {
+    uploadResource: (pointer: PointerFile, resource: Blob) => Promise<string>;
+    matchPath: (path: string) => boolean;
+  },
+  mediaFiles: AssetProxy[],
+) {
+  return await Promise.all(
+    mediaFiles.map(async mediaFile => {
+      const { fileObj, path } = mediaFile;
+      const fixedPath = path.startsWith('/') ? path.slice(1) : path;
+      if (!client.matchPath(fixedPath)) {
+        return mediaFile;
+      }
+
+      const pointerFileDetails = await getPointerFileForMediaFileObj(client, fileObj as File, path);
+      return { ...mediaFile, ...pointerFileDetails };
+    }),
+  );
+}