prick 0.19.0 → 0.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 75523a625d7139830b419f2624f96af3c029dc1ad89b0cb40dec289f4b94b522
-  data.tar.gz: 450c53de9ef6b3c8627df7471076b87c559f367d7aef30e009a846bffa98390c
+  metadata.gz: 2abdd6978f9dce4cf9570d2ad5165fef65541ed5e69bd6150086a30b232498a5
+  data.tar.gz: bb1accf8f541d7197c73b14407401d91d5e3b9475b205bad46bea8851d2825c6
 SHA512:
-  metadata.gz: 98b24c53c2ae54007c46cb4157963ba049327e6ea9b73c5b6b8502367e58d9d018acdb74c9896394bd90a0335359719c0e02523f1e2965de1ed31c023285eeb9
-  data.tar.gz: 25b7425c7f8547bdd0be1ee0a570b1b6115f08ff2ba3bd8c8284cced0ffc5da1ecbc0670e57bb21d2b9a45276df03d0f2f3210b83712ff50f55f811991ba11e3
+  metadata.gz: 951dee9f87d7ba9475bf70512b35d29720a30ec5371e2c03c00575c21a94163f3aa43bf28bfbb6a5fe80e0da63cb1154bd84f29ae671651b7aa54de0e45630e2
+  data.tar.gz: cc5ef462d6c9ae5e1add6bb9fc53ffc4a6684c64020d24fad39327cd80218f01c031060f7c2e14a125a94caad305c658994e2ea53845f211909a4764a72f8d75

data/Gemfile

@@ -1,9 +1,15 @@
-source "https://rubygems.org"
+# frozen_string_literal: true
 
-git_source(:github) {|repo_name| "https://github.com/clrgit/#{repo_name}" }
+source "https://rubygems.org"
 
-# Specify your gem's dependencies in prick.gemspec
+# Specify your gem's dependencies in prick_build.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
+
+gem 'shellopts', :github => 'clrgit/shellopts', branch: "master"
+gem 'pg_graph', :github => 'clrgit/pg_graph', branch: "master"
+gem 'pg_meta', :github => 'clrgit/pg_meta', branch: "master"
+gem 'pg_conn', :github => 'clrgit/pg_conn', branch: "master"
+

data/README.md

@@ -1,6 +1,6 @@
-# Prick
+# PrickBuild
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/prick`. To experiment with that code, run `bin/console` for an interactive prompt.
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/prick_build`. To experiment with that code, run `bin/console` for an interactive prompt.
 
 TODO: Delete this and the text above, and describe your gem
 
@@ -9,16 +9,16 @@ TODO: Delete this and the text above, and describe your gem
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'prick'
+gem 'prick_build'
 ```
 
 And then execute:
 
-    $ bundle
+    $ bundle install
 
 Or install it yourself as:
 
-    $ gem install prick
+    $ gem install prick_build
 
 ## Usage
 
@@ -28,8 +28,8 @@ TODO: Write usage instructions here
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/prick.
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/prick_build.

data/Rakefile

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/TODO

@@ -1,13 +1,15 @@
 
-o Add a top-level 'global' directory for stuff that doesn't belong to a schema or that 
-  require superuser privileges. prick-build should then execute the content of that
-  directory first before switching to the database user
-o Check for commits to tags
-o Accumulate version history in prick.versions instead of just having the newest
-o Using rc's in migration syntax: fork-version-fork-version.rc1 ?
-o Make it possible to execute prick in any subdirecty instead of only in the root
-o Hack migra to create separate file(s) for table changes
-o Also execute subject/ directory if only a subject.sql is found. subject.yml
-  should still override everything
-o Track included files - this will make dependency checks much easier
+o Store fox.state, reflections, and reflections in the database so third-party
+  programs can use them when testing
+o 'prick make <resource>' should build everything up the resource, then commit
+  before trying to build the resource upon the database. Then it is easy to
+  re-run the failed (sql-)script in psql to see why it failed
+o 'prick make' should build everything up a touched file, then commit before
+  proceeding
+o Build functions after schema so that there are no order dependencies on
+  functions. Problem is to detect function files
+o Build views after schema so that there are no order dependencies on
+  views. Problem is to detect view files
 
++ prick release major|minor|patch
++ Check modification time of build.yml files too

data/bin/console

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require "bundler/setup"
-require "prick"
+require "prick_build"
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.

data/doc/build-yml.txt

@@ -0,0 +1,14 @@
+
+Format
+
+---
+- schema: schema-name
+- file
+- seed: file
+- seed:
+    - file
+    - file
+- schema_init: file
+- schema_term: file
+- init: file
+- term: file

data/exe/prick

@@ -1,39 +1,250 @@
 #!/usr/bin/env ruby
 
-require 'prick.rb'
-require 'prick/program.rb'
+#$LOAD_PATH.unshift "/home/clr/prj/shellopts/lib"
 
-require 'shellopts'
+require 'bootsnap'
+begin
+  cache_dir = "/run/user/" + `id -u`.chomp
+  Bootsnap.setup(
+    cache_dir:            cache_dir,            # Path to your cache
+    development_mode:     true,                 # Current working environment, e.g. RACK_ENV, RAILS_ENV, etc
+    load_path_cache:      true,                 # Optimize the LOAD_PATH with a cache
+    compile_cache_iseq:   true,                 # Compile Ruby code into ISeq cache, breaks coverage reporting.
+    compile_cache_yaml:   true                  # Compile YAML into a cache
+  )
+  Bootsnap.instrumentation = ->(event, path) { puts "#{event} #{path}" }
+end
 
-require 'tempfile'
+require 'prick.rb'
+require 'shellopts'
 
 include ShellOpts
 include Prick
 
+TIME = false
 
 SPEC = %(
-  -n,name=NAME
-      Name of project. Defauls to the environment variable `PRICK_PROJECT` if
-      set and else the name of the current directory
-
-  -C,directory=DIR
-      Change to directory DIR before doing anything else
-
-  -h,help COMMAND...
-      Print this page
+  @ Prick project command
 
   +v,verbose
-      Be verbose. Repeated --verbose options increase the verbosity level
+      Be verbose. Repeated -v options increase the verbosity level
 
   -q,quiet
       Be quiet
 
-  --version
-      Print prick version
+  -C,directory=EDIR
+      Change to directory DIR before doing anything else
+
+  -d,database=DATABASE
+      Override database name from prick.yml
+
+  -U,username=USERNAME
+      Override username from from prick.yml
+
+  version!
+      Print project version
+
+  init! -n,name=NAME -t,title=TITLE [DIRECTORY]
+      Initializes a prick project
+
+  setup!
+      Create the database user (if necessary) and an empty database
+
+  teardown!
+      Drop the database and the database user. TODO: Also run teardown scripts
+
+  create.data!
+  create.schema!
+  create.database!
+  create.users!
+  create.all!
+      Create an object. Fails if migration exist unless the --force flag is given
+
+  create.migration! -f,force -o,file=FILE -- VERSION
+      Create a migration from VERSION to the current and write it to
+      migration/VERSION. Fails if migration exist unless the --force flag is
+      given. If --file is given, the migration is written to FILE instead of
+      the migration directory. This doesn't require you to be on a release
+      branch and can be used to create ad-hoc migration scripts
+
+  drop! -- [KIND]
+      @ Drop objects
+
+      Kind can be 'users', 'data', 'schema', 'database' (the default), or 'all'. It is
+      not an error if the object doesn't exist. TODO Only 'users' is currently defined
+
+  build! -t,time --dump=KIND? -- [SCHEMA]
+      Build the project. If SCHEMA is defined, later schemas are excluded.
+      KIND can be 'nodes', 'allnodes' or 'batches' (the default)
+
+  make! -t,time --dump=KIND? -- [SCHEMA]
+      @ Only rebuild changed files
+
+      Checks file timestamps against the time of the last build and only
+      rebuild affected parts of the project. KIND can be 'nodes', 'allnodes' or
+      'batches'
+
+  fox! -- FILE...
+      Load fox file data. Data are reset to their initial state after build
+      before the fox data are loaded
+
+  release! -- KIND
+      Create a release of the given kind. KIND can be 'major', 'minor', or
+      'patch'. Release checks that the current repo is clean and up to date
+      with the origin
+
+  migrate! -f,file=FILE
+      Execute a migration
+
+  dump.type!
+  dump.data!
+  dump.schema!
+  dump.database!
+      TODO
+
+  dump.migration! --force -- VERSION
+)
+
+opts, args = ShellOpts.process(SPEC, ARGV, exception: true)
+
+begin
+  # Handle verbose and quiet
+  $verbose = opts.verbose
+  $quiet = opts.quiet?
+
+  # Honor -C option
+  if opts.directory?
+    if File.exist?(opts.directory)
+      begin
+        Dir.chdir(opts.directory)
+      rescue Errno::ENOENT
+        raise Prick::Error, "Can't cd to '#{opts.directory}'"
+      end
+    else
+      raise Prick::Error, "Can't find directory: #{opts.directory}"
+    end
+  end
+
+  # Get subcommand
+  cmd = opts.subcommand!
+
+  # Process init command
+  if opts.subcommand == :init
+    dir, state = Prick::SubCommand.init(args.expect(0..1), cmd.name, cmd.title, opts.database, opts.username)
+    puts "Initialized prick project '#{state.name}' in #{dir}"
+    if opts.database.nil? || opts.username.nil?
+      puts
+      puts "Please check database/username in #{PRICK_CONTEXT_FILE}"
+    end
+    exit
+  end
+
+  # Check state
+  File.exist?(PRICK_PROJECT_FILE) or raise Prick::Error, "Not in a prick project directory"
+
+  # Load state
+  Prick.state = State.load
+
+  # Handle -d and -U options
+  database = opts.database || Prick.state.database
+  username = opts.username || Prick.state.username
+    
+  # Expect a sub-command
+  cmd = opts.subcommand! or raise Prick::Error, "Subcomand expected"
+
+  # Process subcommands
+  case opts.subcommand
+    when :version!
+      puts "#{Prick.state.name}-#{Prick.state.version}"
+
+    when :setup!
+      Prick::SubCommand.setup(database, username)
+
+    when :teardown!
+      Prick::SubCommand.teardown(database, username)
+
+    when :create!
+      create_command = opts.create!
+      case create_command.subcommand
+        when :migration
+          arg = args.expect(1)
+          version = PrickVersion.try(arg) or raise Prick::Error, "Illegal version: #{arg}"
+          Prick::SubCommand.create_migration(
+              username, version, 
+              force: create_command.subcommand!.force?, 
+              file: create_command.subcommand!.file)
+        else
+          raise NotImplementedError
+      end
+
+    when :build!
+      dump = cmd.dump("batches")&.to_sym
+      Prick::SubCommand.build(database, username, args.expect(0..1), timer: cmd.time?, dump: dump)
+
+    when :make!
+      dump = cmd.dump("batches")&.to_sym
+      Prick::SubCommand.make(database, username, args.expect(0..1), timer: cmd.time?, dump: dump)
+
+    when :fox!
+      Prick::SubCommand.fox(database, username, args)
 
-  init! -u,user=USER [DIR]
-      Initialize a project in the given directory. DIR defaults to the current
-      directory. The USER is the postgres user, it defaults to the project name
+    when :drop!
+      case subject = args.expect(1).to_sym
+        when :all
+          Prick::SubCommand.drop_all(database)
+        when :users
+          Prick::SubCommand.drop_users(database)
+        when :database
+          Prick::SubCommand.drop_database(database)
+        when :data, :schema, :database, :all
+          raise NotImplementedError
+      else
+        raise Prick::Error, "Unknown subject: #{subject}"
+      end
+
+    when :release!
+      kind = args.expect(1).to_sym
+      constrain? kind, :major, :minor, :patch or 
+          raise Prick::Fail, "Expected 'major', 'minor', or 'patch' argument, got '#{kind}'"
+      Prick::SubCommand.release(kind)
+
+    when :migrate!
+      args.expect(0)
+      Prick::SubCommand.migrate(database, username, file: cmd.file)
+
+    when :dump!
+      subject = cmd.subcommand!
+      case cmd.subcommand
+        when :migration
+          arg = args.expect(1)
+          version = PrickVersion.try(arg) or raise "Illegal version number: #{arg}"
+          Prick::SubCommand.create_migration(username, version, force: subject.force?, file: "/dev/stdout")
+        when :data, :schema, :database
+          raise NotImplementedError
+      else
+        raise Prick::Error, "Unknown subject: #{subject}"
+      end
+
+    else
+      raise Prick::Fail, "Internal error: Unhandled command - #{opts.subcommand.inspect}"
+  end
+
+rescue ShellOpts::Failure, Prick::Fail, Prick::Build::PostgresError => ex 
+  ShellOpts.failure(ex.message)
+
+rescue ShellOpts::Error, Prick::Error => ex
+  ShellOpts.error(ex.message)
+end
+
+__END__
+
+  -n,name=NAME
+      Name of project. Defauls to the environment variable `PRICK_PROJECT` if
+      set and else the name of the current directory
+
+  init! -u,user=USER [NAME]
+      Initialize a project in the given directory. The USER is the postgres
+      user and defaults to the project name
 
   info!
       Print project information
@@ -160,6 +371,13 @@ SPEC = %(
       in the var/spool directory
 )
 
+__END__
+  
+
+
+
+
+
 DEFAULT_STATE_FILE = "fox.state"
 
 opts, args = ShellOpts.process(SPEC, ARGV)

data/lib/builder/batch.rb

@@ -0,0 +1,147 @@
+module Prick
+  module Build
+    class BuildBatch
+      include Timer
+
+      attr_reader :builder
+      forward_to :builder, :conn
+
+      def kind() @nodes.first&.kind end
+      attr_reader :nodes
+
+      def initialize(builder) 
+        @builder = builder
+        @nodes = [] 
+      end
+
+      def execute(&block)
+        name = self.class.to_s.sub(/.*::/, "").split(/(?=[A-Z])/).map(&:downcase).join(" ")
+        time "Execute #{name} (nodes: #{nodes.size})" do
+          yield
+        end
+      end
+
+      def dump
+        puts kind.upcase
+        indent { nodes.each(&:dump) }
+      end
+    end
+
+    class SqlBatch < BuildBatch
+      def execute
+        super {
+          begin
+            sql = []
+            if nodes.first.is_a?(ExeNode)
+              time "Execute script" do
+                sql = [nodes.first.source]
+              end
+            end
+            conn.execute sql + nodes[sql.size..-1].map(&:source)
+
+          rescue PG::SyntaxError, PG::Error => ex 
+            error, line, pos = parse_pg_message(ex.message)
+            if line
+              if line < nodes.first.lines
+                node = nodes.first;
+              else
+                line -= nodes.first.lines
+                node = nodes[1..-1].find { |node|
+                  line -= 1
+                  if line < node.lines
+                    line -= 2
+                    true
+                  else
+                    line -= node.lines
+                    false
+                  end
+                }
+              end
+
+              if node.is_a?(SqlNode)
+                message = ["prick: #{error} in #{node.path}", line, pos].compact.join(":")
+              else
+                message = "prick: #{error} from #{node.path}"
+              end
+            else
+              raise
+            end
+            raise PostgresError.new(message)
+          end
+        }
+      end
+
+    private
+      # Returns [error, line, pos] tuple
+      def parse_pg_message(message)
+        message =~ /ERROR:\s*(.*?)\n(LINE (\d+): ).*?\n(\s*\^)/m or return nil
+        [$1, $3.to_i, $4.size - $2.size]
+      end
+    end
+
+    class ModuleBatch < BuildBatch
+      def execute
+        super { 
+          nodes.each { |node| 
+            sql = node&.call
+            conn.execute sql if sql 
+          }
+        }
+      end
+    end
+
+    class FoxBatch < BuildBatch
+      # Inlining Fox instead of using command line program as it saves some deciseconds
+      def execute
+        t_type = t_fox = t_exe = nil
+
+        super {
+          timer = Timer.new "Execute fox batch (nodes: #{nodes.size}, clean: #{builder.clean})"
+          t_type = Timer.new "  Load data model"
+
+          files = nodes.map(&:path)
+
+          # Load meta object
+          meta = PgMeta.new(conn)
+
+          # Create type object
+          type = PgGraph::Type.new(meta, builder.reflections_file)
+
+          t_type.stop
+          t_fox = Timer.new "  Parse files"
+
+          # Create fox object
+          fox = FixtureFox::Fox.new(type)
+
+          # Parse files
+          for file in files
+            source, lines = fox.tokenize(file)
+            fox.parse(source, lines)
+          end
+
+          # Analyze
+          fox.assign_types
+          fox.generate
+
+          # Dump state file
+          fox.write_state(FOX_STATE_PATH)
+
+          t_fox.stop
+          t_exe = Timer.new "  Execute SQL"
+
+          delete = builder.clean ? :none : :touched
+          begin
+            conn.execute fox.to_sql(format: :exec, delete: delete)
+            conn.execute "update prick.versions set built_at = now() at time zone 'UTC'"
+          rescue PG::SyntaxError, PG::Error => ex 
+            raise PostgresError, "prick: #{error} from #{node.path}"
+          end
+        }
+        t_type.emit
+        t_fox.emit
+        t_exe.emit
+      end
+    end
+  end
+end
+

data/lib/builder/builder.rb

@@ -0,0 +1,122 @@
+require 'local/command'
+
+require 'builder/node.rb'
+require 'builder/node_pool.rb'
+require 'builder/batch.rb'
+require 'builder/parser.rb'
+
+include Constrain
+
+module Prick
+  module Build
+    class Error < StandardError; end
+    class PostgresError < Error; end
+
+    class Builder
+      # PgConn object
+      attr_reader :conn
+
+      # Root schema directory
+      attr_reader :dir
+
+      # Reflections YAML file
+      attr_reader :reflections_file
+
+      # True if database is initially clean - ie. all tables are empty
+      attr_accessor :clean
+
+      # Root build node
+      attr_reader :root
+
+      # Pool of nodes. Initialized by #load_pool
+      attr_reader :pool
+
+      forward_to :pool, :nodes, :decl_nodes, :init_nodes, :term_nodes,
+                        :seed_nodes, :fox_seed_nodes, :sql_seed_nodes,
+                        :schemas
+
+      def batches() @batches ||= group end
+
+      def initialize(conn, dir, clean = true, touched: false)
+        @conn = conn
+        @dir = dir
+        @reflections_file = REFLECTIONS_PATH
+        @clean = clean
+        @pool = NodePool.new
+        @root = Parser.parse(conn, dir)
+        load_pool(@root) # Collect nodes into pool
+        @batches = nil # Initialized by #group
+      end
+
+      # Group sources into batches
+      def group
+        @batches = []
+        kind = nil
+        batch = nil
+
+        for node in [init_nodes, decl_nodes, fox_seed_nodes, sql_seed_nodes, term_nodes].flatten
+          case node.kind
+            when :module
+              if batch&.kind != :module
+                @batches << batch if batch
+                batch = ModuleBatch.new(self)
+              end
+            when :exe # Exe sources always create a new batch
+              @batches << batch if batch
+              batch = SqlBatch.new(self)
+            when batch&.kind
+              ;
+            when :sql || node.kind == :inline
+              if batch&.kind != :exe
+                @batches << batch if batch
+                batch = SqlBatch.new(self)
+              end
+            when :inline
+              @batches << batch if batch
+              batch = SqlBatch.new(self)
+            when :fox
+              @batches << batch if batch
+              batch = FoxBatch.new(self)
+            when :yml
+              next
+          else
+            raise Error, "Unexpected node kind: #{node.kind}"
+          end
+          batch.nodes << node
+        end
+        @batches << batch if batch
+      end
+
+      def execute(conn, create_schemas: schemas)
+        group if batches.nil?
+        conn.exec create_schemas.grep_v("public").map { |schema| "create schema #{schema}" }
+        for batch in batches
+          batch.execute
+        end
+      end
+
+#     def setup
+#     end
+
+#     def teardown
+#     end
+
+      def dump
+        batches ? batches.each(&:dump) : pool.dump
+      end
+
+    private
+      def load_pool(build_node)
+        pool.add(build_node.init_nodes)
+        build_node.decl_nodes.each { |node| 
+          pool.add node
+          load_pool(node) if node.kind == :yml
+        }
+        pool.add(build_node.seed_nodes)
+        pool.add(build_node.term_nodes)
+      end
+    end
+  end
+end
+
+