duck_duck_duck 1.0.0

data/node/migrate_old

@@ -0,0 +1,205 @@
+#!/usr/bin/env node
+
+var _ = require('underscore')
+;
+
+// Create databases: okdoki
+
+var River  = require('da_river').River
+, Topogo   = require('topogo').Topogo
+, Customer = require('../Server/Customer/model').Customer
+, Chat_Bot = require('../Server/Chat/Chat_Bot').Chat_Bot
+, h        = require('../test/helpers')
+;
+
+var _      = require('underscore');
+
+var cmd         = (process.argv[2] || 'nothing')
+, is_reset_user = cmd === 'reset_with_data'
+, is_reset      = cmd === 'reset' || is_reset_user
+, is_up         = is_reset || cmd === 'up'
+, is_down       = is_reset || cmd === 'down'
+;
+
+if (!is_up && !is_down) {
+  console.log('Unknown cmd: ' + process.argv[2]);
+  process.exit(1);
+}
+
+var ok = {
+  list : [],
+  q: function (string) {
+    this.list.push(string
+                   .replace(/\$id_size/g, Topogo.id_size)
+                   .replace(/\$trashed_at/, " trashed_at  bigint  default null ")
+                  );
+    return this;
+  }
+};
+
+function down(names, flow) {
+  var public = [];
+  var r = River.new(arguments);
+
+  if (!is_down)
+    return flow.finish(public);
+
+  _.each(names, function (n, i) {
+    if (n.indexOf('public.') === 0 ) {
+      public.push(n);
+      r.job('drop', n, function (j) {
+        Topogo.new(n).drop(j);
+      });
+    };
+  });
+
+  r.job('public tables', function (j, last) {
+    return j.finish(public);
+  });
+
+  r.run();
+}
+
+function up(flow) {
+  if (!is_up)
+    return flow.finish();
+
+  // ok.q("CREATE EXTENSION IF NOT EXISTS pgcrypto");
+
+  // ok.q(" \
+       // CREATE OR REPLACE FUNCTION public.json_get_varchar_array(j text, k text) \
+       // RETURNS varchar[] \
+       // AS $$    \
+       // import json; \
+       // d = json.loads(j or '{}'); \
+       // return d[k] if k in d else []; \
+       // $$ LANGUAGE plpython3u; \
+  // ");
+
+  // ok.q(" \
+       // CREATE OR REPLACE FUNCTION public.json_get_text_array(j text, k text) \
+       // RETURNS text[] \
+       // AS $$    \
+       // import json; \
+       // d = json.loads(j or '{}')[k]; \
+       // return  d[k] if k in d else []; \
+       // $$ LANGUAGE plpython3u; \
+  // ");
+
+   // ok.q(" \
+        // CREATE OR REPLACE FUNCTION public.json_get(j text, k text) \
+        // RETURNS text \
+        // AS $$    \
+        // import json; \
+        // d = json.loads(j or '{}')[k]; \
+        // return  d[k] if k in d else \"\"; \
+        // $$ LANGUAGE plpython3u; \
+  // ");
+
+     // ok.q(" \
+          // CREATE OR REPLACE FUNCTION public.json_merge(o text, n text) \
+          // RETURNS text \
+          // AS $$    \
+          // import json; \
+          // oj = json.loads(o or \"{}\");   \
+          // nj = json.loads(n or \"{}\");   \
+          // f  = dict(list(oj.items()) + list(nj.items())); \
+          // return json.dumps(f);         \
+          // $$ LANGUAGE plpython3u; \
+ // ");
+
+// ok.q(" \
+// CREATE OR REPLACE FUNCTION encode_pass_phrase(varchar) \
+// RETURNS varchar \
+// AS $$ \
+// SELECT encode(digest($1, 'sha512'), 'hex') \
+// $$ LANGUAGE SQL STRICT IMMUTABLE; \
+// ");
+
+
+} // end func up
+
+function create(flow) {
+  console.log('Finished migrating the main db.');
+  if (!is_reset_user) {
+    return flow.finish();
+  }
+
+  var p     = "pass phrase";
+  var report = function (j) {
+    console.log('Finished ' + j.group + ' ' + j.id);
+  };
+
+  var c_opts = {pass_phrase: p, confirm_pass_phrase: p, ip: '000.000.00'};
+
+  var r = River.new(arguments);
+  r.for_each_finish(report);
+  r.job('create:', 'go99', function (j) {
+    Customer.create(_.extend({screen_name: j.id}, c_opts), (j));
+  });
+
+  r.job('create:', 'dos', function (j) {
+    Customer.create(_.extend({screen_name: j.id}, c_opts), (j));
+  });
+
+  r.job('create bot:', '404', function (j) {
+    var c = j.river.reply_for('create:', 'go99');
+    Chat_Bot.create({owner_id: c.data.id, url: "https://okdoki-bot.herokuapp.com/test/404/404", name: j.id}, (j));
+  });
+
+  r.job('create bot:', 'ok', function (j) {
+    var c = j.river.reply_for('create:', 'go99');
+    Chat_Bot.create({owner_id: c.data.id, url: "https://okdoki-bot.herokuapp.com/test/ok", name: j.id}, (j));
+  });
+
+  r.job('create bot:', 'im', function (j) {
+    var c = j.river.reply_for('create:', 'dos');
+    Chat_Bot.create({owner_id: c.data.id, url: "https://okdoki-bot.herokuapp.com/test/im", name: j.id}, (j));
+  });
+
+  r.job('create bot:', 'not_json', function (j) {
+    var c = j.river.reply_for('create:', 'dos');
+    Chat_Bot.create({owner_id: c.data.id, url: "https://okdoki-bot.herokuapp.com/test/not_json", name: j.id}, (j));
+  });
+
+  r.run();
+
+};
+
+// // ==========================================================================================
+
+// // ==========================================================================================
+
+
+// ****************************************************************
+// ****************** SQL Helpers *********************************
+// ****************************************************************
+
+var trashed_at = " trashed_at bigint default NULL ";
+
+function varchar(o) { return " varchar( " + o + " ) "; }
+var primary_key = " PRIMARY KEY ";
+var serial      = " serial ";
+var not_null    = " NOT NULL ";
+var unique      = " unique ";
+var bigint      = " bigint ";
+var default_null= "default null";
+
+function create_table(name, f) {
+  var vals = [];
+  var sql = function () {
+    vals.push(_.toArray(arguments));
+  };
+
+  f(sql);
+
+  return "CREATE TABLE IF NOT EXISTS " + name + " ( \n" + to_sql(vals) + " \n ); ";
+}
+
+function to_sql(vals) {
+  var lines = _.map(vals, function (row) { return row.join( ' ' ); });
+  return lines.join(", \n");
+}
+
+
+

data/node/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "duck_duck_duck",
+  "version": "0.4.4",
+  "description": "My personal way to do multi-applet migrations.",
+  "main": "app.js",
+  "directories": {
+    "test": "test"
+  },
+  "scripts": {
+    "test": "bin/test"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/da99/duck_duck_duck.git"
+  },
+  "dependencies": {
+    "underscore": "x.x.x",
+    "optimist": "x.x.x",
+    "da_river": "x.x.x",
+    "topogo": "x.x.x"
+  },
+  "keywords": [
+    "da99"
+  ],
+  "author": "da99",
+  "license": "MIT",
+  "readmeFilename": "README.md",
+  "gitHead": "849c5374d165e9fb10721becc31538c6f610ef59"
+}

data/node/template.js

@@ -0,0 +1,22 @@
+
+var Topogo = require("topogo").Topogo;
+var River  = require("da_river").River;
+
+var table = "";
+var m     = module.exports = {};
+
+m.migrate = function (dir, r) {
+
+  if (dir === 'down') {
+
+    Topogo.run('DROP TABLE IF EXISTS ' + table +  ';', [], r);
+
+  } else {
+
+    var sql = 'CREATE TABLE IF NOT EXISTS ' + table + " ( \
+\
+    );";
+    Topogo.run(sql, [], r);
+
+  }
+};

data/node/test.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# -*- bash -*-
+#
+set -u -e -o pipefail
+
+export SCHEMA_TABLE="_test_schema"
+export DUCK_TEMPLATE="../../template.js" 
+
+rm -f /tmp/duck_*
+touch /tmp/duck_up
+touch /tmp/duck_down
+touch /tmp/duck_drop_it
+
+function init {
+  rm -f "migrates/002-two.js"
+  rm -f "migrates/003-three.js"
+  rm -f "migrates/004-four.js"
+  rm -f "migrates/005-five.js"
+  rm -f "migrates/006-six.js"
+
+  cp "migrates/001-one.js"  "migrates/002-two.js"
+  cp "migrates/001-one.js"  "migrates/003-three.js"
+}
+
+function init_last_three {
+  cp "migrates/001-one.js"  "migrates/004-four.js"
+  cp "migrates/001-one.js"  "migrates/005-five.js"
+  cp "migrates/001-one.js"  "migrates/006-six.js"
+}
+
+# ==== reset
+node tests/helpers/drop.js
+
+cd tests/user
+../../bin/duck_duck_duck up
+
+cd ../raven_sword
+../../bin/duck_duck_duck up
+
+cd ../praying_mantis
+rm -f "migrates/008-eight.js"
+rm -f "migrates/010-ten.js"
+cp "migrates/002-two.js"  "migrates/004-four.js"
+cp "migrates/002-two.js"  "migrates/006-six.js"
+../../bin/duck_duck_duck up
+cp "migrates/002-two.js"  "migrates/008-eight.js"
+cp "migrates/002-two.js"  "migrates/010-ten.js"
+../../bin/duck_duck_duck down
+
+cd ../lone_wolf
+init
+../../bin/duck_duck_duck up
+init_last_three
+../../bin/duck_duck_duck up
+
+cd ../laughing_octopus
+rm -rf migrates
+../../bin/duck_duck_duck create one
+../../bin/duck_duck_duck create two
+../../bin/duck_duck_duck create three
+
+# test .drop/.create
+cd ../screaming_mantis
+../../bin/duck_duck_duck up
+../../bin/duck_duck_duck down
+
+cd ../liquid
+../../bin/duck_duck_duck up
+../../bin/duck_duck_duck drop_it
+
+cd ../..
+bin/duck_duck_duck list > /tmp/duck_list
+mocha tests/duck_duck_duck.js
+
+
+

data/node/tests.js

@@ -0,0 +1,161 @@
+
+var _    = require('underscore')
+, Topogo = require('topogo').Topogo
+, River  = require('da_river').River
+, assert = require('assert')
+, fs     = require('fs')
+;
+
+var does = function (name, func) {
+  if (func.length !== 1)
+    throw new Error('Test func requires done: ' + name);
+  return it(name, func);
+};
+
+var applets_list = {};
+function applets(func) {
+  if (applets_list.length)
+    return func(applets_list);
+
+  River.new()
+  .job(function (j) {
+    Topogo.run('SELECT * FROM _test_schema', [], j);
+  })
+  .run(function (r, recs) {
+    _.each(recs, function (r, i) {
+      applets_list[r.name] = r.version;
+    });
+    func(applets_list);
+  });
+}
+
+describe( 'Before first migrate:', function () {
+
+  it( 'creates schema table', function (done) {
+    River.new(null)
+    .job(function (j) {
+      Topogo.run('SELECT * FROM _test_schema', [], j);
+    })
+    .job(function (j, last) {
+      assert.equal(last.length > 0, true);
+      done();
+    })
+    .run();
+  });
+
+  it( 'creates rows with: name, version', function (done) {
+    River.new(null)
+    .job(function (j) {
+      Topogo.run('SELECT * FROM _test_schema', [], j);
+    })
+    .job(function (j, last) {
+      assert.deepEqual(_.keys(last[0]), ['name', 'version']);
+      done();
+    })
+    .run();
+  });
+
+}); // === end desc
+
+describe( 'Migrate up:', function () {
+
+  does( 'updates version to latest migrate', function (done) {
+    River.new()
+
+    .job(function (j) {
+      Topogo.run('SELECT * FROM _test_schema', [], j);
+    })
+
+    .job(function (j, last) {
+      assert(last[0].version, 3);
+      done();
+    })
+
+    .run();
+  });
+
+  it( 'migrates files higher, but not equal, of current version', function () {
+    var contents = fs.readFileSync('/tmp/duck_up').toString().trim();
+    assert.equal(contents, "+1+2+3+4+5+6");
+  });
+
+}); // === end desc
+
+describe( 'Migrate down:', function () {
+
+  var contents = null;
+
+  before(function () {
+    contents = fs.readFileSync('/tmp/duck_down').toString().trim();
+  });
+
+  it( 'runs migrates in reverse order', function () {
+    assert.equal(contents, "+2+4+6-6-4-2");
+  });
+
+  it( 'does not run down migrates from later versions', function () {
+    // This tests is the same as "runs migrates in reverse order"
+    assert.equal(contents, "+2+4+6-6-4-2");
+  });
+
+  does( 'update version to one less than earlier version', function (done) {
+    River.new(null)
+    .job(function (j) {
+      Topogo.run('SELECT * FROM _test_schema', [], j);
+    })
+    .job(function (j, last) {
+      var pm = _.find(last, function (rec) {
+        return rec.name === 'praying_mantis';
+      });
+      assert.equal(pm.version, 0);
+      done();
+    })
+    .run();
+  });
+
+}); // === end desc
+
+
+describe( 'create ...', function () {
+
+  it( 'create a file', function () {
+    var contents = fs.readFileSync("tests/laughing_octopus/migrates/001-one.js").toString();
+    assert.equal(contents.indexOf('var ') > -1, true);
+  });
+
+  it( 'creates file in successive order', function () {
+    var contents = fs.readFileSync("tests/laughing_octopus/migrates/002-two.js").toString();
+    assert.equal(contents.indexOf('var ') > -1, true);
+    var contents = fs.readFileSync("tests/laughing_octopus/migrates/003-three.js").toString();
+    assert.equal(contents.indexOf('var ') > -1, true);
+  });
+
+}); // === end desc
+
+
+
+describe( 'drop_it', function () {
+
+  it( 'migrates down', function () {
+    var contents = fs.readFileSync("/tmp/duck_drop_it").toString();
+    assert.deepEqual(contents, "drop_it");
+  });
+
+  does( 'removes entry from schema', function (done) {
+    applets(function (list) {
+      assert.deepEqual(list.liquid, undefined);
+      done();
+    });
+  });
+
+}); // === end desc
+
+
+describe( 'list', function () {
+
+  it( 'outputs schema table on each line: VER_NUM NAME', function () {
+    var contents = fs.readFileSync("/tmp/duck_list").toString().split("\n");
+    assert.deepEqual(!!contents[0].match(/\d user/), true);
+    assert.deepEqual(!!contents[1].match(/\d raven_sword/), true);
+  });
+}); // === end desc

data/specs/duck_duck_duck.rb

@@ -0,0 +1,150 @@
+
+require 'sequel'
+require 'Exit_0'
+
+schema = ENV['SCHEMA_TABLE'] = '_test_schema'
+DB = Sequel.connect ENV['DATABASE_URL']
+MODELS = Dir.glob('*/migrates').map { |dir|  File.basename File.dirname(dir) }
+
+# === Reset tables ===========================================================
+def reset
+  tables = MODELS + [ENV['SCHEMA_TABLE']]
+  tables.each { |t|
+    DB << "DROP TABLE IF EXISTS #{t.inspect};"
+  }
+end
+
+reset
+
+# === Helpers ================================================================
+def get *args
+  field = args.last.is_a?(Symbol) ? args.pop : nil
+  rows = DB[*args].all
+  if field
+    rows.map { |row| row[field] }
+  else
+    rows
+  end
+end
+
+def versions mod
+  Dir.glob("#{mod}/migrates/*").map { |file|
+    file[/\/(\d{4})[^\/]+\.sql$/] && $1.to_i
+  }.sort
+end
+
+# === Specs ==================================================================
+
+describe "create" do
+
+  before {
+    tmp_dir = '/tmp/ddd_ver'
+    Exit_0("rm -fr #{tmp_dir}")
+    Exit_0("mkdir -p #{tmp_dir}")
+    @dir = tmp_dir
+  }
+
+  it "names the file in successive file versions: 0000-....sql" do
+    Dir.chdir(@dir) {
+      Exit_0("duck_duck_duck create MOD table_1")
+      Exit_0("duck_duck_duck create MOD table_2")
+
+      Exit_0("touch MOD/migrates/0022-skip_zero.sql")
+      Exit_0("duck_duck_duck create MOD table_3")
+
+      Exit_0("touch MOD/migrates/0091-skip_zero.sql")
+      Exit_0("duck_duck_duck create MOD table_100")
+
+      File.should.exists('MOD/migrates/0010-table_1.sql')
+      File.should.exists('MOD/migrates/0020-table_2.sql')
+      File.should.exists('MOD/migrates/0030-table_3.sql')
+      File.should.exists('MOD/migrates/0100-table_100.sql')
+    }
+  end
+
+end # === describe create
+
+describe 'up model' do
+
+  before { reset }
+
+  it( 'updates version to latest migration' ) do
+    Exit_0("duck_duck_duck up 0010_model")
+    get('SELECT * FROM _test_schema').
+      first[:version].should == versions('0010_model').last
+  end
+
+  it 'does not run migrations from previous versions' do
+    Exit_0("duck_duck_duck migrate_schema")
+    DB << File.read("0010_model/migrates/0010-table.sql").split('-- DOWN').first
+    DB << "INSERT INTO #{schema.inspect} VALUES ('0010_model', '20');"
+    Exit_0("duck_duck_duck up 0010_model")
+    get('SELECT * FROM "0010_model"', :title).
+      should == ['record 30', 'record 40', 'record 50']
+  end
+
+end # === describe up model
+
+describe 'down model' do
+
+  before { reset }
+
+  it 'leaves version to 0' do
+    Exit_0("duck_duck_duck up 0010_model")
+    Exit_0("duck_duck_duck down 0010_model")
+    get(%^SELECT * FROM #{schema.inspect} WHERE name = '0010_model'^, :version).last.
+      should == 0
+  end
+
+  it 'runs migrates in reverse order' do
+    Exit_0("duck_duck_duck up 0020_model")
+    Exit_0("duck_duck_duck down 0020_model")
+    get('SELECT * FROM "0020_model"', :title).
+      should == ['record 20', 'record 30', 'DROP record 30', 'DROP record 20', 'DROP 0020_model']
+  end
+
+  it 'does not run down migrates from later versions' do
+    Exit_0("duck_duck_duck migrate_schema")
+    DB << File.read("0020_model/migrates/0010-table.sql").split('-- DOWN').first
+    DB << "INSERT INTO #{schema.inspect} VALUES ('0020_model', '20');"
+    DB << "UPDATE #{schema} SET version = '20' WHERE name = '0020_model';"
+    Exit_0("duck_duck_duck down 0020_model")
+    get('SELECT * FROM "0020_model"', :title).
+      should == ['DROP record 20', 'DROP 0020_model']
+  end
+
+end # === describe down model
+
+describe "up" do
+
+  before { reset }
+
+  it "migrates all models" do
+    Exit_0("duck_duck_duck up")
+    get("SELECT * FROM #{schema.inspect} ORDER BY name").
+      should == [
+        {:name=>'0010_model',:version=>50},
+        {:name=>'0020_model',:version=>30},
+        {:name=>'0030_model',:version=>30}
+    ]
+  end
+
+end # describe up
+
+describe 'down' do
+
+  before { reset }
+
+  it "migrates down all models" do
+    Exit_0("duck_duck_duck up")
+    Exit_0("duck_duck_duck down")
+    get("SELECT * FROM #{schema.inspect} ORDER BY name").
+      should == [
+        {:name=>'0010_model',:version=>0},
+        {:name=>'0020_model',:version=>0},
+        {:name=>'0030_model',:version=>0}
+    ]
+  end
+
+end # === describe down
+

data/specs/lib/helpers.rb

@@ -0,0 +1,4 @@
+
+require 'Bacon_Colored'
+require 'duck_duck_duck'
+require 'pry'

data/specs/lib/models/0010_model/migrates/0010-table.sql

@@ -0,0 +1,13 @@
+
+CREATE TABLE "0010_model" (
+  id serial NOT NULL PRIMARY KEY,
+  title varchar(100)
+);
+
+
+-- DOWN
+
+INSERT INTO "0010_model" (title)
+VALUES ('DROP 0010_model');
+
+

data/specs/lib/models/0010_model/migrates/0020-insert_1.sql

@@ -0,0 +1,11 @@
+
+INSERT INTO "0010_model" (title)
+VALUES ('record 20');
+
+
+-- DOWN
+
+INSERT INTO "0010_model" (title)
+VALUES ('DROP record 20: 0010_model');
+
+

data/specs/lib/models/0010_model/migrates/0030-insert_2.sql

@@ -0,0 +1,11 @@
+
+INSERT INTO "0010_model" (title)
+VALUES ('record 30');
+
+
+-- DOWN
+
+INSERT INTO "0010_model" (title)
+VALUES ('DROP record 30');
+
+

data/specs/lib/models/0010_model/migrates/0040-insert_3.sql

@@ -0,0 +1,11 @@
+
+INSERT INTO "0010_model" (title)
+VALUES ('record 40');
+
+
+-- DOWN
+
+INSERT INTO "0010_model" (title)
+VALUES ('DROP record 40');
+
+