gitsheets 0.21.1 → 0.21.5

package/bin/cli.js

@@ -2,18 +2,22 @@
 
 
 // setup logger
-const logger = require('winston')
-module.exports = { logger }
+const logger = require('winston');
+const loggerConsole = new logger.transports.Console({
+  level: process.env.DEBUG ? 'debug' : 'info',
+  format: logger.format.combine(
+    logger.format.colorize(),
+    logger.format.prettyPrint(),
+    logger.format.splat(),
+    logger.format.simple(),
+  ),
 
-if (process.env.DEBUG) {
-  logger.level = 'debug'
-}
+  // all logger output to STDERR
+  stderrLevels: Object.keys(require('winston/lib/winston/config').cli.levels),
+});
+logger.add(loggerConsole);
 
-
-// all logger output to STDERR
-for (const level in logger.levels) {
-  logger.default.transports.console.stderrLevels[level] = true;
-}
+module.exports = { logger };
 
 
 // route command line
@@ -25,15 +29,33 @@ require('yargs')
     default: false,
     global: true,
   })
+  .option('q', {
+    alias: 'quiet',
+    type: 'boolean',
+    default: false,
+    global: true,
+  })
   .check(function (argv) {
     if (argv.debug) {
-      logger.level = 'debug'
+      loggerConsole.level = 'debug';
     } else if (argv.quiet) {
-      logger.level = 'error'
+      loggerConsole.level = 'error';
     }
 
     return true;
   })
-  .commandDir('../commands')
+  .commandDir('../commands', { exclude: /\.test\.js$/ })
   .demandCommand()
-  .argv
+  .showHelpOnFail(false, 'Specify --help for available options')
+  .fail((msg, err) => {
+    logger.error(msg || err.message);
+
+    if (err) {
+      logger.debug(err.stack);
+    }
+
+    process.exit(1);
+  })
+  .strict()
+  .help()
+  .argv;

package/commands/edit.js

@@ -19,6 +19,8 @@ exports.builder = {
 exports.handler = async function edit({ recordPath, resumePath, encoding }) {
   const fs = require('fs');
   const { spawn } = require('child_process');
+  const path = require('path');
+  const tmp = require('tmp');
   const TOML = require('@iarna/toml');
   const Repository = require('../lib/Repository.js');
   const Sheet = require('../lib/Sheet.js')
@@ -26,17 +28,21 @@ exports.handler = async function edit({ recordPath, resumePath, encoding }) {
   const git = await repo.getGit();
 
   // open record
-  const recordToml = fs.readFileSync(resumePath || recordPath, encoding);
+  let recordToml = fs.readFileSync(resumePath || recordPath, encoding);
 
-  // get temp path
-  const tempFilePath = await new Promise((resolve, reject) => {
-    const mktemp = spawn('mktemp', ['-t', 'gitsheet.XXXXXX.toml']);
-
-    let stdout = '', stderr = '';
-    mktemp.stdout.on('data', chunk => stdout += chunk);
-    mktemp.stderr.on('data', chunk => stderr += chunk);
+  // try to parse and format
+  try {
+    const record = TOML.parse(recordToml);
+    recordToml = Sheet.stringifyRecord(record);
+  } catch (err) {
+    console.warn(`Failed to parse opened record:\n${err}`);
+  }
 
-    mktemp.on('close', code => code === 0 ? resolve(stdout.trim()) : reject(stderr.trim()));
+  // get temp path
+  const { name: tempFilePath } = tmp.fileSync({
+    prefix: path.basename(recordPath, '.toml'),
+    postfix: '.toml',
+    discardDescriptor: true,
   });
 
   // populate temp path
@@ -67,7 +73,7 @@ exports.handler = async function edit({ recordPath, resumePath, encoding }) {
   try {
     editedRecord = TOML.parse(editedToml);
   } catch (err) {
-    console.error(`Failed to parse record:\n${err}`);
+    console.error(`Failed to parse edited record:\n${err}`);
     console.error(`To resume editing, run: git sheet edit ${recordPath} ${tempFilePath}`);
     process.exit(1);
   }

package/commands/normalize.js

@@ -54,9 +54,23 @@ exports.handler = async function query({
     }
 
     // loop through all records and re-upsert
-    for await (const record of sheet.query()) {
-      logger.info(`rewriting ${sheetName}/${record[Symbol.for('gitsheets-path')]}`);
-      await sheet.upsert(record);
+    try {
+      for await (const record of sheet.query()) {
+        const originalPath = record[Symbol.for('gitsheets-path')];
+        logger.info(`rewriting ${path.join(root, prefix, sheetName, originalPath)}.toml`);
+        const { path: normalizedPath } = await sheet.upsert(record);
+
+        if (normalizedPath !== originalPath) {
+          logger.warn(`^- moved to ${path.join(root, prefix, sheetName, normalizedPath)}.toml`);
+        }
+      }
+    } catch (err) {
+      if (err.constructor.name == 'TomlError') {
+        logger.error(`failed to parse ${path.join(root, prefix, err.file)}\n${err.message}`);
+        process.exit(1);
+      }
+
+      throw err;
     }
   }
 

package/commands/read.js

@@ -3,7 +3,7 @@ exports.desc = 'Read a record, converting to desired format';
 exports.builder = {
   'record-path': {
     type: 'string',
-    describe: 'The path to a record file to edit',
+    describe: 'The path to a record file to read',
     demandOption: true,
   },
   encoding: {
@@ -24,12 +24,7 @@ exports.builder = {
 
 exports.handler = async function edit({ recordPath, encoding, format, headers }) {
   const fs = require('fs');
-  const { spawn } = require('child_process');
   const TOML = require('@iarna/toml');
-  const Repository = require('../lib/Repository.js');
-  const Sheet = require('../lib/Sheet.js')
-  const repo = await Repository.getFromEnvironment({ working: true });
-  const git = await repo.getGit();
 
   // open record
   const recordToml = fs.readFileSync(recordPath, encoding);
@@ -64,6 +59,6 @@ async function outputCsv(record, { headers = true, delimiter = ',' } = {}) {
 }
 
 async function outputToml(record) {
-  const TOML = require('@iarna/toml');
-  console.log(`${TOML.stringify(record)}`);
+  const Sheet = require('../lib/Sheet.js')
+  console.log(`${Sheet.stringifyRecord(record)}`);
 }

package/commands/singer-target.js

@@ -101,11 +101,20 @@ exports.handler = async function singerTarget({
   const clearedSheets = new Set();
   const writtenStreams = new Set();
   for await (const { type, stream, ...message} of readMessages({ jsonlFile })) {
-    const sheet = sheets[stream];
-
     console.log(`${type}\t${stream}`, message);
 
 
+    // ignore unhandled message types for now
+    if (type == 'STATE' || type == 'ACTIVATE_VERSION') {
+      console.warn(`ignoring ${type} message`);
+      continue;
+    }
+
+
+    // get sheet
+    const sheet = sheets[stream];
+
+
     // create schema if needed
     if (!sheet) {
       if (type == 'SCHEMA') {
@@ -129,13 +138,6 @@ exports.handler = async function singerTarget({
     }
 
 
-    // ignore state for now
-    if (type == 'STATE') {
-      console.warn('ignoring STATE message');
-      continue;
-    }
-
-
     // handle record message
     if (type == 'RECORD') {
       if (deleteMissing && !clearedSheets.has(sheet)) {

package/lib/Sheet.js

@@ -118,9 +118,20 @@ class Sheet extends Configurable
   async readRecord (blob, path = null) {
     const cache = this.#recordCache.get(blob.hash);
 
-    const record = cache
-      ? v8.deserialize(cache)
-      : await blob.read().then(TOML.parse);
+    let record;
+
+    if (cache) {
+      record = v8.deserialize(cache);
+    } else {
+      const toml = await blob.read();
+
+      try {
+        record = TOML.parse(toml);
+      } catch (err) {
+        err.file = path;
+        throw err;
+      }
+    }
 
     // annotate with gitsheets keys
     record[RECORD_SHEET_KEY] = this.name;
@@ -153,14 +164,14 @@ class Sheet extends Configurable
     const pathTemplate = PathTemplate.fromString(pathTemplateString);
     const sheetDataTree = await this.dataTree.getSubtree(sheetRoot);
 
-    BLOBS: for await (const blob of pathTemplate.queryTree(sheetDataTree, query)) {
-      const record = await this.readRecord(blob);
+    BLOBS: for await (const { blob, path: blobPath } of pathTemplate.queryTree(sheetDataTree, query)) {
+      const record = await this.readRecord(blob, blobPath);
 
       if (!queryMatches(query, record)) {
         continue BLOBS;
       }
 
-      record[RECORD_PATH_KEY] = pathTemplate.render(record);
+      record[RECORD_PATH_KEY] = blobPath || pathTemplate.render(record);
 
       yield record;
     }

package/lib/path/Template.js

@@ -50,7 +50,7 @@ class Template
     return recordPath.join('/');
   }
 
-  async* queryTree (tree, query, depth = 0) {
+  async* queryTree (tree, query, pathPrefix = '', depth = 0) {
     const numComponents = this.#components.length;
 
     if (!tree) {
@@ -68,7 +68,7 @@ class Template
           const child = await currentTree.getChild(`${nextName}.toml`);
 
           if (child) {
-            yield child;
+            yield { path: path.join(pathPrefix, nextName), blob: child };
           }
 
           // absolute match on a leaf, we're done with this query
@@ -97,8 +97,9 @@ class Template
               continue;
             }
 
-            attachmentsPrefix = `${childPath.substr(0, childPath.length - 5)}/`;
-            yield child;
+            const childName = childPath.substr(0, childPath.length - 5);
+            attachmentsPrefix = `${childName}/`;
+            yield { path: path.join(pathPrefix, childName), blob: child };
           }
 
           return;
@@ -117,14 +118,14 @@ class Template
       } else {
         // each tree in current tree could contain matching records
         const children = await currentTree.getChildren();
-        for (const childName in children) {
-          const child = children[childName];
+        for (const childPath in children) {
+          const child = children[childPath];
 
           if (!child.isTree) {
             continue;
           }
 
-          yield* this.queryTree(child, query, i+1);
+          yield* this.queryTree(child, query, path.join(pathPrefix, childPath), i+1);
         }
 
         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitsheets",
-  "version": "0.21.1",
+  "version": "0.21.5",
   "description": "A toolkit for using a git repository to store low-volume, high-touch, human-scale data",
   "main": "lib/GitSheets.js",
   "scripts": {
@@ -26,7 +26,7 @@
     "fast-csv": "^4.3.6",
     "fast-json-patch": "^2.2.1",
     "get-stream": "^5.2.0",
-    "hologit": "^0.40.1",
+    "hologit": "^0.40.5",
     "http-assert": "^1.4.1",
     "koa": "^2.13.1",
     "koa-bodyparser": "^4.3.0",
@@ -36,8 +36,9 @@
     "rfc6902": "^4.0.1",
     "sort-keys": "^4.2.0",
     "streaming-json-stringify": "^3.1.0",
+    "tmp": "^0.2.1",
     "to-readable-stream": "^2.1.0",
-    "winston": "^2.4.5",
+    "winston": "^3.3.3",
     "yargs": "^13.3.2"
   },
   "devDependencies": {