gitsheets 0.20.0 → 0.21.3

package/bin/cli.js

@@ -2,18 +2,22 @@
 
 
 // setup logger
-const logger = require('winston')
-module.exports = { logger }
+const logger = require('winston');
+const loggerConsole = new logger.transports.Console({
+  level: process.env.DEBUG ? 'debug' : 'info',
+  format: logger.format.combine(
+    logger.format.colorize(),
+    logger.format.prettyPrint(),
+    logger.format.splat(),
+    logger.format.simple(),
+  ),
 
-if (process.env.DEBUG) {
-  logger.level = 'debug'
-}
+  // all logger output to STDERR
+  stderrLevels: Object.keys(require('winston/lib/winston/config').cli.levels),
+});
+logger.add(loggerConsole);
 
-
-// all logger output to STDERR
-for (const level in logger.levels) {
-  logger.default.transports.console.stderrLevels[level] = true;
-}
+module.exports = { logger };
 
 
 // route command line
@@ -25,15 +29,33 @@ require('yargs')
     default: false,
     global: true,
   })
+  .option('q', {
+    alias: 'quiet',
+    type: 'boolean',
+    default: false,
+    global: true,
+  })
   .check(function (argv) {
     if (argv.debug) {
-      logger.level = 'debug'
+      loggerConsole.level = 'debug';
     } else if (argv.quiet) {
-      logger.level = 'error'
+      loggerConsole.level = 'error';
     }
 
     return true;
   })
-  .commandDir('../commands')
+  .commandDir('../commands', { exclude: /\.test\.js$/ })
   .demandCommand()
-  .argv
+  .showHelpOnFail(false, 'Specify --help for available options')
+  .fail((msg, err) => {
+    logger.error(msg || err.message);
+
+    if (err) {
+      logger.debug(err.stack);
+    }
+
+    process.exit(1);
+  })
+  .strict()
+  .help()
+  .argv;

package/commands/edit.js

@@ -19,6 +19,8 @@ exports.builder = {
 exports.handler = async function edit({ recordPath, resumePath, encoding }) {
   const fs = require('fs');
   const { spawn } = require('child_process');
+  const path = require('path');
+  const tmp = require('tmp');
   const TOML = require('@iarna/toml');
   const Repository = require('../lib/Repository.js');
   const Sheet = require('../lib/Sheet.js')
@@ -26,17 +28,21 @@ exports.handler = async function edit({ recordPath, resumePath, encoding }) {
   const git = await repo.getGit();
 
   // open record
-  const recordToml = fs.readFileSync(resumePath || recordPath, encoding);
+  let recordToml = fs.readFileSync(resumePath || recordPath, encoding);
 
-  // get temp path
-  const tempFilePath = await new Promise((resolve, reject) => {
-    const mktemp = spawn('mktemp', ['-t', 'gitsheet.XXXXXX.toml']);
-
-    let stdout = '', stderr = '';
-    mktemp.stdout.on('data', chunk => stdout += chunk);
-    mktemp.stderr.on('data', chunk => stderr += chunk);
+  // try to parse and format
+  try {
+    const record = TOML.parse(recordToml);
+    recordToml = Sheet.stringifyRecord(record);
+  } catch (err) {
+    console.warn(`Failed to parse opened record:\n${err}`);
+  }
 
-    mktemp.on('close', code => code === 0 ? resolve(stdout.trim()) : reject(stderr.trim()));
+  // get temp path
+  const { name: tempFilePath } = tmp.fileSync({
+    prefix: path.basename(recordPath, '.toml'),
+    postfix: '.toml',
+    discardDescriptor: true,
   });
 
   // populate temp path
@@ -67,7 +73,7 @@ exports.handler = async function edit({ recordPath, resumePath, encoding }) {
   try {
     editedRecord = TOML.parse(editedToml);
   } catch (err) {
-    console.error(`Failed to parse record:\n${err}`);
+    console.error(`Failed to parse edited record:\n${err}`);
     console.error(`To resume editing, run: git sheet edit ${recordPath} ${tempFilePath}`);
     process.exit(1);
   }

package/commands/normalize.js

@@ -54,9 +54,23 @@ exports.handler = async function query({
     }
 
     // loop through all records and re-upsert
-    for await (const record of sheet.query()) {
-      logger.info(`rewriting ${sheetName}/${record[Symbol.for('gitsheets-path')]}`);
-      await sheet.upsert(record);
+    try {
+      for await (const record of sheet.query()) {
+        const originalPath = record[Symbol.for('gitsheets-path')];
+        logger.info(`rewriting ${path.join(root, prefix, sheetName, originalPath)}.toml`);
+        const { path: normalizedPath } = await sheet.upsert(record);
+
+        if (normalizedPath !== originalPath) {
+          logger.warn(`^- moved to ${path.join(root, prefix, sheetName, normalizedPath)}.toml`);
+        }
+      }
+    } catch (err) {
+      if (err.constructor.name == 'TomlError') {
+        logger.error(`failed to parse ${path.join(root, prefix, err.file)}\n${err.message}`);
+        process.exit(1);
+      }
+
+      throw err;
     }
   }
 

package/commands/read.js

@@ -3,7 +3,7 @@ exports.desc = 'Read a record, converting to desired format';
 exports.builder = {
   'record-path': {
     type: 'string',
-    describe: 'The path to a record file to edit',
+    describe: 'The path to a record file to read',
     demandOption: true,
   },
   encoding: {
@@ -24,12 +24,7 @@ exports.builder = {
 
 exports.handler = async function edit({ recordPath, encoding, format, headers }) {
   const fs = require('fs');
-  const { spawn } = require('child_process');
   const TOML = require('@iarna/toml');
-  const Repository = require('../lib/Repository.js');
-  const Sheet = require('../lib/Sheet.js')
-  const repo = await Repository.getFromEnvironment({ working: true });
-  const git = await repo.getGit();
 
   // open record
   const recordToml = fs.readFileSync(recordPath, encoding);
@@ -64,6 +59,6 @@ async function outputCsv(record, { headers = true, delimiter = ',' } = {}) {
 }
 
 async function outputToml(record) {
-  const TOML = require('@iarna/toml');
-  console.log(`${TOML.stringify(record)}`);
+  const Sheet = require('../lib/Sheet.js')
+  console.log(`${Sheet.stringifyRecord(record)}`);
 }

package/commands/upsert.js

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const TOML = require('@iarna/toml');
 const { parse: csvParse } = require('fast-csv');
+const deepmerge = require('deepmerge');
 
 const inputFormats = {
   json: readJsonFile,
@@ -46,6 +47,11 @@ exports.builder = {
     type: 'boolean',
     default: false,
   },
+  'patch-existing': {
+    describe: 'For existing records, patch in provided values so that additional properties not included in the input are preserved',
+    type: 'boolean',
+    default: false,
+  },
 };
 
 exports.handler = async function upsert({
@@ -57,6 +63,7 @@ exports.handler = async function upsert({
   encoding,
   attachments = null,
   deleteMissing,
+  patchExisting,
   ...argv
 }) {
   const logger = require('../lib/logger.js');
@@ -96,7 +103,13 @@ exports.handler = async function upsert({
 
 
   // clear sheet
+  let inputSheet = sheet;
+
   if (deleteMissing) {
+    // re-open input sheet
+    inputSheet = await sheet.clone();
+
+    // clear target sheet
     await sheet.clear();
   }
 
@@ -113,8 +126,28 @@ exports.handler = async function upsert({
 
 
   // upsert record(s) into sheet
-  for await (const inputRecord of inputRecords) {
-    const { blob: outputBlob, path: outputPath } = await sheet.upsert(inputRecord);
+  for await (let inputRecord of inputRecords) {
+
+    if (patchExisting) {
+      // TODO: move more of this logic inside Sheet class
+
+      // fetch existing record from inputSheet
+      const inputRecordPath = await inputSheet.pathForRecord(await inputSheet.normalizeRecord(inputRecord));
+
+      if (inputRecordPath) {
+        const { root: inputSheetRoot } = await inputSheet.getCachedConfig();
+
+        // existing record find, merge
+        const existingBlob = await inputSheet.dataTree.getChild(`${path.join(inputSheetRoot, inputRecordPath)}.toml`);
+
+        if (existingBlob) {
+          const existingRecord = await inputSheet.readRecord(existingBlob);
+          inputRecord = deepmerge(inputRecord, existingRecord);
+        }
+      }
+    }
+
+    const { blob: outputBlob, path: outputPath } = await sheet.upsert(inputRecord, { patchExisting });
     console.log(`${outputBlob.hash}\t${outputPath}`);
 
     if (attachments) {

package/lib/Sheet.js

@@ -118,9 +118,20 @@ class Sheet extends Configurable
   async readRecord (blob, path = null) {
     const cache = this.#recordCache.get(blob.hash);
 
-    const record = cache
-      ? v8.deserialize(cache)
-      : await blob.read().then(TOML.parse);
+    let record;
+
+    if (cache) {
+      record = v8.deserialize(cache);
+    } else {
+      const toml = await blob.read();
+
+      try {
+        record = TOML.parse(toml);
+      } catch (err) {
+        err.file = path;
+        throw err;
+      }
+    }
 
     // annotate with gitsheets keys
     record[RECORD_SHEET_KEY] = this.name;
@@ -153,14 +164,14 @@ class Sheet extends Configurable
     const pathTemplate = PathTemplate.fromString(pathTemplateString);
     const sheetDataTree = await this.dataTree.getSubtree(sheetRoot);
 
-    BLOBS: for await (const blob of pathTemplate.queryTree(sheetDataTree, query)) {
-      const record = await this.readRecord(blob);
+    BLOBS: for await (const { blob, path: blobPath } of pathTemplate.queryTree(sheetDataTree, query)) {
+      const record = await this.readRecord(blob, blobPath);
 
       if (!queryMatches(query, record)) {
         continue BLOBS;
       }
 
-      record[RECORD_PATH_KEY] = pathTemplate.render(record);
+      record[RECORD_PATH_KEY] = blobPath || pathTemplate.render(record);
 
       yield record;
     }
@@ -277,6 +288,15 @@ class Sheet extends Configurable
     return this.dataTree.writeChild(root, this.dataTree.repo.createTree());
   }
 
+  async clone () {
+    return new Sheet({
+      workspace: this.workspace,
+      name: this.name,
+      dataTree: await this.dataTree.clone(),
+      configPath: this.configPath,
+    });
+  }
+
   async upsert (record) {
     const {
       root: sheetRoot,

package/lib/path/Template.js

@@ -50,7 +50,7 @@ class Template
     return recordPath.join('/');
   }
 
-  async* queryTree (tree, query, depth = 0) {
+  async* queryTree (tree, query, pathPrefix = '', depth = 0) {
     const numComponents = this.#components.length;
 
     if (!tree) {
@@ -68,7 +68,7 @@ class Template
           const child = await currentTree.getChild(`${nextName}.toml`);
 
           if (child) {
-            yield child;
+            yield { path: path.join(pathPrefix, nextName), blob: child };
           }
 
           // absolute match on a leaf, we're done with this query
@@ -97,8 +97,9 @@ class Template
               continue;
             }
 
-            attachmentsPrefix = `${childPath.substr(0, childPath.length - 5)}/`;
-            yield child;
+            const childName = childPath.substr(0, childPath.length - 5);
+            attachmentsPrefix = `${childName}/`;
+            yield { path: path.join(pathPrefix, childName), blob: child };
           }
 
           return;
@@ -117,14 +118,14 @@ class Template
       } else {
         // each tree in current tree could contain matching records
         const children = await currentTree.getChildren();
-        for (const childName in children) {
-          const child = children[childName];
+        for (const childPath in children) {
+          const child = children[childPath];
 
           if (!child.isTree) {
             continue;
           }
 
-          yield* this.queryTree(child, query, i+1);
+          yield* this.queryTree(child, query, path.join(pathPrefix, childPath), i+1);
         }
 
         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitsheets",
-  "version": "0.20.0",
+  "version": "0.21.3",
   "description": "A toolkit for using a git repository to store low-volume, high-touch, human-scale data",
   "main": "lib/GitSheets.js",
   "scripts": {
@@ -22,10 +22,11 @@
   "dependencies": {
     "@iarna/toml": "^2.2.5",
     "csv-parser": "^2.3.5",
-    "fast-csv": "^3.7.0",
+    "deepmerge": "^4.2.2",
+    "fast-csv": "^4.3.6",
     "fast-json-patch": "^2.2.1",
     "get-stream": "^5.2.0",
-    "hologit": "^0.34.3",
+    "hologit": "^0.40.5",
     "http-assert": "^1.4.1",
     "koa": "^2.13.1",
     "koa-bodyparser": "^4.3.0",
@@ -35,15 +36,16 @@
     "rfc6902": "^4.0.1",
     "sort-keys": "^4.2.0",
     "streaming-json-stringify": "^3.1.0",
+    "tmp": "^0.2.1",
     "to-readable-stream": "^2.1.0",
-    "winston": "^2.4.5",
+    "winston": "^3.3.3",
     "yargs": "^13.3.2"
   },
   "devDependencies": {
     "common-tags": "^1.8.0",
     "del": "^5.1.0",
     "into-stream": "^5.1.1",
-    "jest": "^24.9.0",
+    "jest": "^26.6.3",
     "make-dir": "^3.1.0",
     "supertest": "^4.0.2"
   }