@alexbosworth/fiat 2.0.0 → 3.0.0

package/chain/blocks/get_blocks_for_date.js

@@ -0,0 +1,110 @@
+const asyncAuto = require('async/auto');
+const {returnResult} = require('asyncjs-util');
+
+const {confirmationsAfterDay} = require('./../constants');
+const findHeightForTime = require('./find_height_for_time');
+const {isCalendarDay} = require('./../../dates');
+const {msPerSecond} = require('./../constants');
+
+const dayFormat = /^\d{4}-\d{2}-\d{2}$/;
+const isHeight = n => Number.isInteger(n) && n >= 0;
+const range = (a, b) => [...Array(b - a).keys()].map(i => a + i);
+const secondsPerDay = 60 * 60 * 24;
+
+/** Get the heights of the blocks that belong to a UTC calendar day
+
+  {
+    date: <Day YYYY-MM-DD String>
+    height: <Current Best Chain Height Number>
+    lnd: <Authenticated LND API Object>
+    [times]: <Block Times By Height Map Object>
+  }
+
+  @returns via cbk or Promise
+  {
+    heights: [<Block Height Number>]
+    is_complete: <Day Is Complete Bool>
+  }
+*/
+module.exports = (args, cbk) => {
+  return new Promise((resolve, reject) => {
+    return asyncAuto({
+      // Check arguments
+      validate: cbk => {
+        if (!args.date || !dayFormat.test(args.date)) {
+          return cbk([400, 'ExpectedYyyyMmDdDateToGetBlocksForDate']);
+        }
+
+        if (!isCalendarDay({day: args.date}).is_calendar_day) {
+          return cbk([400, 'ExpectedValidDateToGetBlocksForDate']);
+        }
+
+        if (!isHeight(args.height)) {
+          return cbk([400, 'ExpectedCurrentChainHeightToGetBlocksForDate']);
+        }
+
+        if (!args.lnd) {
+          return cbk([400, 'ExpectedAuthenticatedLndToGetBlocksForDate']);
+        }
+
+        return cbk();
+      },
+
+      // Determine the start and end times of the day
+      times: ['validate', ({}, cbk) => {
+        const start = Date.parse(`${args.date}T00:00:00.000Z`) / msPerSecond;
+
+        return cbk(null, {start, end: start + secondsPerDay});
+      }],
+
+      // Block times are kept to avoid retrieving them twice
+      known: ['validate', ({}, cbk) => cbk(null, args.times || new Map())],
+
+      // Find the first block on the day
+      getFirst: ['known', 'times', ({known, times}, cbk) => {
+        return findHeightForTime({
+          height: args.height,
+          lnd: args.lnd,
+          low: Number(),
+          time: times.start,
+          times: known,
+        },
+        cbk);
+      }],
+
+      // Find the block after the last block on the day
+      getNext: [
+        'getFirst',
+        'known',
+        'times',
+        ({getFirst, known, times}, cbk) =>
+      {
+        return findHeightForTime({
+          height: args.height,
+          lnd: args.lnd,
+          low: getFirst.at_or_after,
+          time: times.end,
+          times: known,
+        },
+        cbk);
+      }],
+
+      // Determine the blocks on the day
+      blocks: ['getFirst', 'getNext', ({getFirst, getNext}, cbk) => {
+        const first = getFirst.at_or_after;
+        const next = getNext.at_or_after;
+
+        const heights = range(first, next);
+
+        // A day is complete when it has blocks and enough blocks after it
+        const confirmed = next + confirmationsAfterDay <= args.height + 1;
+
+        return cbk(null, {
+          heights,
+          is_complete: !!heights.length && confirmed,
+        });
+      }],
+    },
+    returnResult({reject, resolve, of: 'blocks'}, cbk));
+  });
+};

package/chain/blocks/get_median_time.js

@@ -0,0 +1,92 @@
+const asyncAuto = require('async/auto');
+const asyncMapLimit = require('async/mapLimit');
+const {returnResult} = require('asyncjs-util');
+
+const blockHeaderTime = require('./block_header_time');
+const {fetchLimit} = require('./../constants');
+const getBlockHeader = require('./get_block_header');
+
+const {floor} = Math;
+const isHeight = n => Number.isInteger(n) && n >= 0;
+const {max} = Math;
+const medianWindowBlocks = 11;
+const range = (a, b) => [...Array(b - a).keys()].map(i => a + i);
+
+/** Get the median time past of a block
+
+  {
+    height: <Block Height Number>
+    lnd: <Authenticated LND API Object>
+    [times]: <Block Times By Height Map Object>
+  }
+
+  @returns via cbk or Promise
+  {
+    time: <Median Time Past Epoch Seconds Number>
+  }
+*/
+module.exports = ({height, lnd, times}, cbk) => {
+  return new Promise((resolve, reject) => {
+    return asyncAuto({
+      // Check arguments
+      validate: cbk => {
+        if (!isHeight(height)) {
+          return cbk([400, 'ExpectedBlockHeightToGetMedianTime']);
+        }
+
+        if (!lnd) {
+          return cbk([400, 'ExpectedAuthenticatedLndToGetMedianTime']);
+        }
+
+        return cbk();
+      },
+
+      // Get the times of the block and the blocks before it
+      getTimes: ['validate', ({}, cbk) => {
+        const known = times || new Map();
+        const start = max(Number(), height - medianWindowBlocks + 1);
+
+        const heights = range(start, height + 1);
+
+        return asyncMapLimit(heights, fetchLimit, (height, cbk) => {
+          // Exit early when the time of the block is already known
+          if (known.has(height)) {
+            return cbk(null, known.get(height));
+          }
+
+          return getBlockHeader({height, lnd}, (err, res) => {
+            if (!!err) {
+              return cbk(err);
+            }
+
+            const {time} = (() => {
+              try {
+                return blockHeaderTime({header: res.header});
+              } catch (err) {
+                return {};
+              }
+            })();
+
+            // Exit early when the header has no time to read
+            if (time === undefined) {
+              return cbk([503, 'ExpectedBlockHeaderToGetMedianTime']);
+            }
+
+            known.set(height, time);
+
+            return cbk(null, time);
+          });
+        },
+        cbk);
+      }],
+
+      // The median time is the middle of the sorted times
+      median: ['getTimes', ({getTimes}, cbk) => {
+        const sorted = getTimes.slice().sort((a, b) => a - b);
+
+        return cbk(null, {time: sorted[floor(sorted.length / 2)]});
+      }],
+    },
+    returnResult({reject, resolve, of: 'median'}, cbk));
+  });
+};

package/chain/blocks/index.js

@@ -0,0 +1,19 @@
+const blockHeaderHash = require('./block_header_hash');
+const findCompleteDay = require('./find_complete_day');
+const getBlockHash = require('./get_block_hash');
+const getBlockPayments = require('./get_block_payments');
+const getBlocksForDate = require('./get_blocks_for_date');
+const getMedianTime = require('./get_median_time');
+const sampleHeights = require('./sample_heights');
+const windowHeights = require('./window_heights');
+
+module.exports = {
+  blockHeaderHash,
+  findCompleteDay,
+  getBlockHash,
+  getBlockPayments,
+  getBlocksForDate,
+  getMedianTime,
+  sampleHeights,
+  windowHeights,
+};

package/chain/blocks/payment_outputs_from_block.js

@@ -0,0 +1,178 @@
+const {compactIntAsNumber} = require('@alexbosworth/blockchain');
+
+const {blockHeaderByteLength} = require('./../constants');
+const {maxTokens} = require('./../constants');
+const {minTokens} = require('./../constants');
+
+const byteCountHash = 32;
+const byteCountInt32 = 4;
+const byteCountInt64 = 8;
+const byteCountMarkerFlag = 2;
+const coinbaseInputIndex = 0xffffffff;
+const hexAsBuffer = hex => Buffer.from(hex, 'hex');
+const isHex = n => !!n && !(n.length % 2) && /^[0-9A-F]*$/i.test(n);
+const maxInputsCount = 5;
+const maxWitnessBytes = 500;
+const opReturn = 0x6a;
+const paymentOutputsCount = 2;
+const witnessFlagValue = 1;
+const witnessMarkerValue = 0;
+
+/** Derive payment-like outputs from a raw block for on-chain price analysis
+
+  {
+    block: <Hex Encoded Raw Block String>
+  }
+
+  @throws
+  <Error>
+
+  @returns
+  {
+    transactions: [{
+      outputs: [<Output Tokens Number>]
+    }]
+  }
+*/
+module.exports = ({block}) => {
+  if (!isHex(block)) {
+    throw new Error('ExpectedHexEncodedBlockToDerivePaymentOutputs');
+  }
+
+  const buffer = hexAsBuffer(block);
+
+  // Exit early with error when the block header data is cut short
+  if (buffer.length <= blockHeaderByteLength) {
+    throw new Error('ExpectedBlockHeaderAndTransactionsToDeriveOutputs');
+  }
+
+  const transactions = [];
+
+  // A compact int count of transactions follows the block header
+  const txCount = compactIntAsNumber({
+    encoded: buffer,
+    start: blockHeaderByteLength,
+  });
+
+  const readCount = () => {
+    const count = compactIntAsNumber({encoded: buffer, start: cursor});
+
+    cursor += count.bytes;
+
+    return count.number;
+  };
+
+  let cursor = blockHeaderByteLength + txCount.bytes;
+  let remaining = txCount.number;
+
+  // Iterate forward through the block, parsing each transaction
+  while (remaining--) {
+    const outputs = [];
+
+    let hasOpReturn = false;
+    let isCoinbase = false;
+    let isOverWitnessLimit = false;
+
+    cursor += byteCountInt32;
+
+    // SegWit transactions have a marker and flag following the version
+    const isSegWit = buffer[cursor] === witnessMarkerValue &&
+      buffer[cursor + 1] === witnessFlagValue;
+
+    if (isSegWit) {
+      cursor += byteCountMarkerFlag;
+    }
+
+    const inputsCount = readCount();
+
+    for (let i = 0; i < inputsCount; i++) {
+      const hash = buffer.subarray(cursor, cursor + byteCountHash);
+
+      cursor += byteCountHash;
+
+      const vout = buffer.readUInt32LE(cursor);
+
+      cursor += byteCountInt32;
+
+      // A coinbase input spends a null outpoint
+      if (vout === coinbaseInputIndex && hash.every(n => !n)) {
+        isCoinbase = true;
+      }
+
+      const scriptLength = readCount();
+
+      cursor += scriptLength + byteCountInt32;
+    }
+
+    const outputsCount = readCount();
+
+    for (let i = 0; i < outputsCount; i++) {
+      const tokens = Number(buffer.readBigUInt64LE(cursor));
+
+      cursor += byteCountInt64;
+
+      const scriptLength = readCount();
+
+      if (!!scriptLength && buffer[cursor] === opReturn) {
+        hasOpReturn = true;
+      }
+
+      cursor += scriptLength;
+
+      outputs.push(tokens);
+    }
+
+    // SegWit places a witness stack for every input after the outputs
+    if (isSegWit) {
+      for (let i = 0; i < inputsCount; i++) {
+        const itemsCount = readCount();
+
+        let witnessBytes = 0;
+
+        for (let j = 0; j < itemsCount; j++) {
+          const itemLength = readCount();
+
+          cursor += itemLength;
+
+          witnessBytes += itemLength;
+        }
+
+        if (witnessBytes > maxWitnessBytes) {
+          isOverWitnessLimit = true;
+        }
+      }
+    }
+
+    cursor += byteCountInt32;
+
+    // Exit early with error when the transaction runs past the block data
+    if (cursor > buffer.length) {
+      throw new Error('UnexpectedTransactionLengthInBlockToDeriveOutputs');
+    }
+
+    // Exit early when the transaction does not look like a regular payment
+    if (isCoinbase || hasOpReturn || isOverWitnessLimit) {
+      continue;
+    }
+
+    if (inputsCount > maxInputsCount || outputsCount !== paymentOutputsCount) {
+      continue;
+    }
+
+    const tokens = outputs.filter(n => n > minTokens && n < maxTokens);
+
+    // Exit early when no output is in the analyzed range
+    if (!tokens.length) {
+      continue;
+    }
+
+    transactions.push({outputs: tokens});
+  }
+
+  // Exit early with error when there is extra data after the transactions
+  if (cursor !== buffer.length) {
+    throw new Error('UnexpectedDataAfterBlockTransactionsToDeriveOutputs');
+  }
+
+  return {transactions};
+};

package/chain/blocks/sample_heights.js

@@ -0,0 +1,33 @@
+const {floor} = Math;
+const range = n => [...Array(n).keys()];
+
+/** Pick evenly spaced heights to get a quick sample of a range of blocks
+
+  {
+    count: <Sample Blocks Count Number>
+    heights: [<Block Height Number>]
+    [initial]: <Blocks In the First Sample Count Number>
+  }
+
+  @returns
+  {
+    heights: [<Sampled Block Height Number>]
+  }
+*/
+module.exports = ({count, heights, initial}) => {
+  // Exit early when there are not more blocks than the sample size
+  if (heights.length <= count) {
+    return {heights};
+  }
+
+  // A doubled sample takes the start of each stretch to keep the first one
+  const isDoubled = !!initial && count > initial;
+
+  // Positions are counted in half stretches so that every sample of a range
+  // maps them to the same heights
+  const halves = range(count).map(i => isDoubled ? 2 * i : 2 * i + 1);
+
+  const indices = halves.map(n => floor(n * heights.length / (2 * count)));
+
+  return {heights: indices.map(index => heights[index])};
+};

package/chain/blocks/window_heights.js

@@ -0,0 +1,34 @@
+const {half} = require('./../constants');
+
+const {ceil} = Math;
+
+/** Split block heights into consecutive windows of blocks
+
+  {
+    count: <Blocks Per Window Count Number>
+    heights: [<Block Height Number>]
+  }
+
+  @returns
+  {
+    windows: [[<Block Height Number>]]
+  }
+*/
+module.exports = ({count, heights}) => {
+  const starts = [...Array(ceil(heights.length / count)).keys()];
+
+  const windows = starts.map(i => heights.slice(i * count, (i + 1) * count));
+
+  const [last] = windows.slice(-1);
+
+  // Exit early when the last window is long enough to stand on its own
+  if (windows.length < 2 || last.length >= count * half) {
+    return {windows};
+  }
+
+  const [beforeLast] = windows.slice(-2);
+
+  const joined = beforeLast.concat(last);
+
+  return {windows: windows.slice(Number(), -2).concat([joined])};
+};

package/chain/chain_rates.js

@@ -0,0 +1,129 @@
+const asyncQueue = require('async/queue');
+
+const {dayOfDate} = require('./../dates');
+const {earliestSupportedDay} = require('./known');
+const {fsMethods} = require('./../storage/constants');
+const {getChainPrice} = require('./estimates');
+const {knownPrices} = require('./known');
+
+const supportedCurrency = 'BTC';
+const supportedFiat = 'USD';
+
+/** Make a method that gets historic USD prices of BTC from chain data
+
+  {
+    fs: {
+      getFile: <Read File Contents Function> (path, cbk) => {}
+      makeDirectory: <Make Directory Function> (path, cbk) => {}
+      renameFile: <Rename File Function> (from, to, cbk) => {}
+      writeFile: <Write File Contents Function> (path, contents, cbk) => {}
+    }
+    lnd: <Authenticated LND API Object>
+    [logger]: <Winston Logger Object>
+    path: <Prices File Path String>
+  }
+
+  @throws
+  <Error>
+
+  @returns
+  {
+    getChainRate: <Get Chain Rate Method> ({
+      currency: <Currency To Price String> // BTC
+      date: <Date To Price ISO 8601 Date String> // priced on its day in UTC
+      fiat: <Fiat Currency To Price In String> // USD
+    }, cbk) => (err, {
+      cents: <US Cents Per BTC on the Day of the Date Number>
+    })
+  }
+*/
+module.exports = ({fs, lnd, logger, path}) => {
+  if (!fs || !fsMethods.every(method => typeof fs[method] === 'function')) {
+    throw new Error('ExpectedFileSystemMethodsToMakeChainRates');
+  }
+
+  if (!lnd) {
+    throw new Error('ExpectedAuthenticatedLndToMakeChainRates');
+  }
+
+  if (!path) {
+    throw new Error('ExpectedPricesFilePathToMakeChainRates');
+  }
+
+  const estimates = {provisional: {}, samples: Number()};
+
+  // A day the data cannot price is not derived again by this method
+  const unsupported = new Map();
+
+  // Lookups are serialized so that each estimate can guide the next one
+  const queue = asyncQueue(({date}, cbk) => {
+    // The shipped prices are read the first time a price is looked up
+    const shipped = (() => {
+      try {
+        return {rates: knownPrices({}).rates};
+      } catch (err) {
+        return {err};
+      }
+    })();
+
+    // Exit early when the shipped prices cannot be read
+    if (!!shipped.err) {
+      return cbk([503, 'FailedToReadKnownPricesToGetChainRate', {
+        err: shipped.err,
+      }]);
+    }
+
+    return getChainPrice({
+      date,
+      estimates,
+      fs,
+      lnd,
+      logger,
+      path,
+      unsupported,
+      known: shipped.rates,
+    },
+    cbk);
+  });
+
+  const getChainRate = ({currency, date, fiat}, cbk) => {
+    if (currency !== supportedCurrency) {
+      return cbk([400, 'UnsupportedCurrencyToGetChainRate']);
+    }
+
+    if (!date || typeof date !== 'string') {
+      return cbk([400, 'ExpectedDateToGetChainRate']);
+    }
+
+    if (fiat !== supportedFiat) {
+      return cbk([400, 'UnsupportedFiatToGetChainRate']);
+    }
+
+    // A date is priced on its day in UTC
+    const {day} = dayOfDate({date});
+
+    // Exit early when the date is not a readable date on a calendar day
+    if (!day) {
+      return cbk([400, 'ExpectedValidDateToGetChainRate']);
+    }
+
+    const earliest = earliestSupportedDay({}).day;
+
+    // Exit early when the date is before the days prices are derived for
+    if (day < earliest) {
+      return cbk([400, 'DateBeforeSupportedRangeToGetChainRate', {
+        earliest_day: earliest,
+      }]);
+    }
+
+    return queue.push({date: day}, (err, res) => {
+      if (!!err) {
+        return cbk(err);
+      }
+
+      return cbk(null, {cents: res.cents});
+    });
+  };
+
+  return {getChainRate};
+};

package/chain/constants.json

@@ -0,0 +1,26 @@
+{
+  "blockHeaderByteLength": 80,
+  "blockHeaderHexLength": 160,
+  "centsPerDollar": 100,
+  "confirmationsAfterDay": 6,
+  "defaultMaxPrice": 10000000,
+  "defaultMinPrice": 100,
+  "fetchLimit": 4,
+  "half": 0.5,
+  "kernelBins": 2,
+  "maxTokens": 10000000000000,
+  "minAliasMargin": 1,
+  "minTokens": 1000,
+  "msPerDay": 86400000,
+  "msPerSecond": 1000,
+  "noiseRadius": 12,
+  "roundTokensMultiple": 1000,
+  "sampleLogInterval": 10,
+  "spikeKernel": [0.3, 0.7, 1, 0.7, 0.3],
+  "tokensPerBigUnit": 100000000,
+  "unpricedErrors": [
+    "ExpectedPaymentOutputsInBlocksToGetPrice",
+    "FailedToDerivePriceFromOutputs",
+    "FailedToDerivePriceInWidenedWindow"
+  ]
+}