prick 0.34.0 → 0.36.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 872c793b29eae6754796a66c93808a58507b5eac112ddadcf2260d7ed01f3435
-  data.tar.gz: 702ee64d93801ddc488664eeae903dc7a34b935585fd87d5c29299295c2e6a7d
+  metadata.gz: 49ed6762f3557cc979f89bb1aaee3d5aa1d8d11a5ba9c7880a8b692d917dc733
+  data.tar.gz: 16436521e2f9a51ed6c2076ba2a54f968735868a711d170253691a69a4b34fb4
 SHA512:
-  metadata.gz: 2524d9c7bbc4f73e5ecb343b5e6ef3caf47a9ed9541d18a96d458dd7024da9915d95b66a90e30dbdcc8e451b7d42cb19b1ae3682d001031ff8d5c09bedfcf354
-  data.tar.gz: a7359ccb249584edd6752509eaa90063d09029c7aa54f879e558644bb2ad2fa261975571f7945e9ea2726eacbdba8a85b7afb3ec8a46c57a0d0f1ec919e5b66f
+  metadata.gz: eedba5e0757b8f19c94d4bf898ea156b93090cd2466e333860d7a4da9ce91d75fbc4acab596bbc06b0a8f260909c9770715299c55040c017d45b7f679d5528f0
+  data.tar.gz: 7d98f2a0a8e6ed8230bc44b3d16d1c46f4baa85956bdbcd893325e2d062d8a6d46c4277cf85af8c5e833fb20bbf8324a75d8788d05418cd87185976c6c16a76a

data/exe/prick

@@ -119,19 +119,19 @@ SPEC = %(
 
       TODO
 
-  build! -f,force -t,time --dump=KIND? -- [SCHEMA]
+  build! --step -f,force -t,time --dump=KIND? -- [SCHEMA]
       Build the project. If SCHEMA is defined, later schemas are excluded. KIND
       can be 'nodes', 'allnodes' or 'batches' (the default). Schemas marked
       with 'no refresh' is not built unless the --force is present
 
-  make! -t,time --dump=KIND? -- [SCHEMA]
+  make! --step -t,time --dump=KIND? -- [SCHEMA]
       @ Only rebuild changed files
 
       Checks file timestamps against the time of the last build and only
       rebuild affected parts of the project. KIND can be 'nodes', 'allnodes' or
       'batches' (the default)
 
-  run! -t,time --dump=KIND? --schema=SCHEMA -- PATH
+  run! --step -t,time --dump=KIND? --schema=SCHEMA -- PATH
       @ Execute path in Prick environment
 
       Execute a single directory or file within the Prick environment. If given
@@ -150,6 +150,12 @@ SPEC = %(
       different data sets
 
       # TODO: A --clean option that resets data
+      #
+
+  snapshot!
+      Records maximum ID for all tables and stores them in prick.snapshot. It is
+      used by 'drop data' to reset data to the point in time of the snapshot.
+      Only one snapshot can be created at a time
 
   version!
       Print project version
@@ -296,7 +302,6 @@ begin
   database = state.database
   username = state.username
   environment = state.environment
-
   # Process subcommands
   case opts.subcommand
     when :version!
@@ -361,27 +366,36 @@ begin
       dump = cmd.dump? ? cmd.dump("batches")&.to_sym || :batches : nil
 #     exclude = cmd.exclude? ? cmd.exclude.split(",") : []
       Prick::SubCommand.build(
-          database, username, args.expect(0..1), force: cmd.force?, timer: cmd.time?, dump: dump)
+          database, username, args.expect(0..1), 
+          force: cmd.force?, step: cmd.step?, timer: cmd.time?, dump: dump)
 
     when :make!
       require_db
       dump = cmd.dump? ? cmd.dump("batches")&.to_sym || :batches : nil
-      Prick::SubCommand.make(database, username, args.expect(0..1), timer: cmd.time?, dump: dump)
+      Prick::SubCommand.make(
+          database, username, args.expect(0..1), 
+          step: cmd.step?, timer: cmd.time?, dump: dump)
 
     when :run!
       require_db
       dump = cmd.dump? ? cmd.dump("batches")&.to_sym || :batches : nil
       Prick::SubCommand.run(
           database, username, args.expect(1), 
-          timer: cmd.time?, dump: dump, schema: cmd.schema)
+          step: cmd.step?, timer: cmd.time?, dump: dump, schema: cmd.schema)
 
     when :bash!
+      args.expect(0)
       Prick::SubCommand.bash(main: cmd.main?)
 
     when :fox!
       require_db
       Prick::SubCommand.fox(database, username, args)
 
+    when :snapshot!
+      require_db
+      args.expect(0)
+      Prick::SubCommand.snapshot(database, username)
+
     when :drop!
       case cmd.subcommand
         when :users!
@@ -397,7 +411,9 @@ begin
           database = args.expect(0..1) || database
           Prick::SubCommand.drop_database(database)
         when :data!
-          raise NotImplementedError
+          args.expect(0)
+          require_db
+          Prick::SubCommand.drop_data(database)
         when :schema!
           require_db
           schemas = args.to_a
@@ -493,7 +509,8 @@ rescue ShellOpts::Error, Prick::Error => ex
   raise
   ShellOpts.error(ex.message)
 
-rescue RuntimeError, IOError, ShellOpts::Failure, Prick::Failure, Prick::Build::PostgresError => ex 
-  ShellOpts.failure(ex.message)
+# FIXME
+#rescue RuntimeError, IOError, ShellOpts::Failure, Prick::Failure, Prick::Build::PostgresError => ex 
+# ShellOpts.failure(ex.message)
 end
 

data/lib/prick/builder/batch.rb

@@ -28,7 +28,8 @@ module Prick
     end
 
     class SqlBatch < BuildBatch
-      def execute(step: true) # This is marginally faster!
+      def execute
+#       puts "#execute"
         super {
           begin
             sql = []
@@ -41,22 +42,16 @@ module Prick
               sql = [node.source]
             end
 
-            if step
-              conn.execute sql, silent: true if !sql.empty?
-              for node in nodes
-                conn.execute node.source, silent: true
-              end
-            else
-              node = nil
-              conn.execute sql + nodes[sql.size..-1].map(&:source), silent: true
+            conn.execute sql, silent: true if !sql.empty?
+            for node in nodes # This is marginally faster than concatenating all nodes
+#             puts node.source
+              conn.execute node.source, silent: true
             end
 
           rescue PG::Error => ex
             error, line, char = conn.err
             file = nil
-            if step
-              ;
-            elsif line
+            if line
               # Locate error node and make line number relative
               if line < nodes.first.lines
                 node = nodes.first
@@ -81,6 +76,7 @@ module Prick
             elsif node
               message = "#{error} from #{node.path}"
             else
+              raise ArgumentError, "Oops"
               message = conn.errmsg + " in SQL batch"
             end
             raise PostgresError.new(message, file, line, char)

data/lib/prick/builder/builder.rb

@@ -45,6 +45,9 @@ module Prick
       # True if this is a single-file build
       attr_reader :single
 
+      # True if SQL entries should be single-stepped. Default false
+      attr_reader :step
+
       # Pool of nodes. Initialized by #load_pool
       attr_reader :pool
 
@@ -55,9 +58,9 @@ module Prick
                         :pg_graph_ignore_schemas, 
                         :refresh_schemas, :keep_schemas
 
-      def batches() @batches ||= group end
+      def batches() @batches || create_batches end
 
-      def initialize(conn, path, clean = true, single: false, touched: false)
+      def initialize(conn, path, clean = true, single: false, touched: false, load_pool: true, step: false)
         File.exist?(path) or raise Error, "Can't find #{path}"
         single || File.directory?(path) or raise Error, "Can't find directory #{path}"
 
@@ -66,14 +69,20 @@ module Prick
         @reflections_file = REFLECTIONS_PATH
         @clean = clean
         @single = single
-        @pool = NodePool.new
+        @step = step
         @root = Parser.parse(conn, path, single: single)
-        load_pool(@root) # Collect nodes into pool
-        @batches = nil # Initialized by #group
+        @pool = nil # Initialized by #load_pool
+        @batches = nil # Initialized by #create_batches
+        self.load_pool if load_pool
+      end
+
+      def load_pool
+        @pool = NodePool.new
+        load_pool_impl(@root)
       end
 
       # Group sources into batches
-      def group
+      def create_batches
         @batches = []
         kind = nil
         batch = nil
@@ -88,8 +97,11 @@ module Prick
             when :exe # Exe sources always create a new batch
               @batches << batch if batch
               batch = SqlBatch.new(self)
-            when batch&.kind
-              ;
+            when batch&.kind # Same kind as current batch
+              if node.kind == :sql && step
+                @batches << batch if batch
+                batch = SqlBatch.new(self)
+              end
             when :sql || node.kind == :inline
               if batch&.kind != :exe
                 @batches << batch if batch
@@ -108,13 +120,17 @@ module Prick
           end
           batch.nodes << node
         end
-
         @batches << batch if batch
+
+        @batches
       end
 
       def execute(conn, create_schemas: schemas)
+        # Load pool
+        load_pool if pool.nil?
+
         # Group batches
-        group if batches.nil?
+        create_batches if batches.nil?
 
         # Register build in database
         Prick.state.save_build_begin
@@ -124,6 +140,10 @@ module Prick
 
         # Execute batch groups
         t0 = Time.now
+#       for batch in batches
+#         puts "Executing #{batch.class} batch"
+#         batch.execute
+#       end
         batches.each(&:execute)
         t1 = Time.now
 
@@ -133,15 +153,16 @@ module Prick
       end
 
       def dump
+        load_pool if pool.nil?
         batches ? batches.each(&:dump) : pool.dump
       end
 
     private
-      def load_pool(build_node)
+      def load_pool_impl(build_node)
         pool.add(build_node.init_nodes)
         build_node.decl_nodes.each { |node| 
           pool.add node
-          load_pool(node) if node.kind == :yml
+          load_pool_impl(node) if node.kind == :yml
         }
         pool.add(build_node.seed_nodes)
         pool.add(build_node.term_nodes)

data/lib/prick/builder/node.rb

@@ -11,7 +11,7 @@ module Prick
       def name() @name = File.basename(path) end
 
       # Note that schema defaults to 'public' which may not be what you want in
-      # all cases
+      # some cases
       def schema() @schema ||= parent&.schema || "public" end
       def schema=(s) @schema = s end
 
@@ -46,7 +46,7 @@ module Prick
 
       def to_s() @to_s ||= [path, args].compact.join(" ") end
       def inspect() to_s end
-      def dump() puts "#{inspect} (#{schema})" end
+      def dump() puts "#{inspect} (#{@schema || 'nil'})" end
 
     protected
       def read_source
@@ -194,7 +194,7 @@ module Prick
         puts "BuildNode #{path}"
         indent {
           puts "has_schema: #{has_schema}"
-          puts "schema: #{schema}" if schema
+          puts "schema: #{@schema || 'nil'}"
           puts "refresh: #{refresh_schema.to_s}"
           puts "pg_graph_ignore_schema: #{pg_graph_ignore_schema}"
           decl_nodes.each(&:dump)

data/lib/prick/builder/parser.rb

@@ -25,7 +25,6 @@ module Prick
       end
 
     private
-
       # First built unit is a RootBuildNode, the rest are regular BuildNode objects
       def make_build_unit(parent, path)
         if @unit
@@ -35,62 +34,13 @@ module Prick
         end
       end
 
-      # Search for an executable in path. Return nil if not found
-      #
-      # Note that "." is ignored in the search path 
-      def find_executable(filename) # ChatGPT
-        Prick.state.executable_search_path.split(File::PATH_SEPARATOR).each do |directory|
-          next if directory == "."
-          path = File.join(directory, filename)
-          return path if File.file?(path) && File.executable?(path)
-        end
-        nil
-      end
-
-      # Expand environment variables in the given file name
-      #
-      # #expend_filename substitute '$<variable>' expressions in the filename
-      # with the corresponding value in the current environment. If the file
-      # was not found, inherited environments are processed hierarchly with the
-      # special environment variable ENVIRONMENT set to each PRICK_ENVIRONMENT
-      # value in the inherited environments
-      #
-      # Return the resulting path to the file and nil if not found
-      #
-      def expand_filename(dir, filename)
-        envs = Prick.state.environments
-        env = envs[Prick.state.environment]
-        bash_vars = Prick.state.bash_environment.dup
-
-        last = nil
-        for env in [env] + env.ancestors.reverse
-          bash_vars["ENVIRONMENT"] = env.name
-          file = expand_variables(filename, bash_vars)
-#         last ||= (file != last and file) or return nil # return if no ENVIRONMENT substitution
-          path = (file.start_with?("/") ? file : File.join(dir, file))
-
-          # Check for file (may be executable)
-          return path if File.exist?(path)
-
-          # Check for executable in search path if file doesn't contain a '/'
-          path = find_executable(file) and return path if file !~ /\//
-        end
-
-        # Return nil if not found
-        return nil
-      end
-
-      # Expand $ENVIRONMENT variable
-      def expand_string(string)
-        expand_variables(string, Prick.state.bash_environment)
-      end
-
-      def parse_path(path)
+      def parse_path(path, schema: nil)
         File.exist?(path) or raise Error, "Can't find #{file}"
         dir = File.dirname(path)
         file = File.basename(path)
         unit = make_build_unit(nil, nil)
-        parse_build_entry(unit, dir, file)
+        unit.schema = schema if schema
+        entry = parse_build_entry(unit, dir, file)
         unit
       end
 
@@ -211,6 +161,56 @@ module Prick
           end
         end
       end
+
+      # Search for an executable in path. Return nil if not found
+      #
+      # Note that "." is ignored in the search path 
+      def find_executable(filename) # ChatGPT
+        Prick.state.executable_search_path.split(File::PATH_SEPARATOR).each do |directory|
+          next if directory == "."
+          path = File.join(directory, filename)
+          return path if File.file?(path) && File.executable?(path)
+        end
+        nil
+      end
+
+      # Expand environment variables in the given file name
+      #
+      # #expend_filename substitute '$<variable>' expressions in the filename
+      # with the corresponding value in the current environment. If the file
+      # was not found, inherited environments are processed hierarchly with the
+      # special environment variable ENVIRONMENT set to each PRICK_ENVIRONMENT
+      # value in the inherited environments
+      #
+      # Return the resulting path to the file and nil if not found
+      #
+      def expand_filename(dir, filename)
+        envs = Prick.state.environments
+        env = envs[Prick.state.environment]
+        bash_vars = Prick.state.bash_environment.dup
+
+        last = nil
+        for env in [env] + env.ancestors.reverse
+          bash_vars["ENVIRONMENT"] = env.name
+          file = expand_variables(filename, bash_vars)
+#         last ||= (file != last and file) or return nil # return if no ENVIRONMENT substitution
+          path = (file.start_with?("/") ? file : File.join(dir, file))
+
+          # Check for file (may be executable)
+          return path if File.exist?(path)
+
+          # Check for executable in search path if file doesn't contain a '/'
+          path = find_executable(file) and return path if file !~ /\//
+        end
+
+        # Return nil if not found
+        return nil
+      end
+
+      # Expand $ENVIRONMENT variable
+      def expand_string(string)
+        expand_variables(string, Prick.state.bash_environment)
+      end
     end
   end
 end

data/lib/prick/share/init/schema/prick/tables.sql

@@ -35,3 +35,12 @@ create table builds (
   make_duration float -- Duration of enclosing make script
 );
 
+create table snapshots (
+  id integer generated by default as identity primary key,
+  schema_name varchar not null,
+  table_name varchar not null,
+  max_id integer, -- may be null
+
+  unique(schema_name, table_name)
+);
+

data/lib/prick/subcommand/prick-build.rb

@@ -7,7 +7,7 @@ module Prick::SubCommand
       database, username, schema, 
       builddir: Prick.state.schema_dir, 
       force: false, # Build all schemas
-      timer: nil, dump: nil)
+      step: false, timer: nil, dump: nil)
 
     Timer.on! if timer
 
@@ -30,7 +30,7 @@ module Prick::SubCommand
           end
           conn = Prick.state.connection
 
-          builder = Prick::Build::Builder.new(conn, builddir)
+          builder = Prick::Build::Builder.new(conn, builddir, step: step)
 
           if exist # Empty (part of) the database
             if force

data/lib/prick/subcommand/prick-drop.rb

@@ -41,6 +41,35 @@ module Prick::SubCommand
     State.connection { |conn| conn.rdbms.drop database }
   end
 
+  # Drop data not in snapshot
+  def self.drop_data(database, schemas = nil)
+    conn = Prick.state.conn
+    conn.session.triggers(false) {
+      if schemas.nil?
+        builder = Prick::Build::Builder.new(conn, Prick.state.schema_dir)
+        pool = builder.pool
+        schemas = pool.refresh_schemas
+      end
+      schema_list = conn.quote_literal_list(schemas)
+      schema_expr = schemas ? "true = true" : "schema_name in #{schema_list}"
+      tables = conn.tuples %(
+        select schema_name, table_name, max_id 
+        from prick.snapshots
+        where #{schema_expr}
+      )
+      tables.each { |schema, table, max_id|
+        uid = "#{schema}.#{table}"
+        if max_id
+          conn.exec "delete from #{uid} where id > #{max_id}"
+          conn.schema.set_serial(schema, table, max_id) if conn.schema.sequence(schema, table)
+        else
+          conn.exec "delete from #{uid}"
+          conn.schema.set_serial(schema, table, nil) if conn.schema.sequence(schema, table)
+        end
+      }
+    }
+  end
+
   # Empty the database
   def self.drop_schema(database, schemas = [])
     constrain database, String

data/lib/prick/subcommand/prick-make.rb

@@ -3,7 +3,7 @@
 require_relative '../builder/builder.rb'
 
 module Prick::SubCommand
-  def self.make(database, username, schema, timer: nil, dump: nil)
+  def self.make(database, username, schema, step: false, timer: nil, dump: nil)
     Timer.on! if timer
     Timer.time "Prick::Command#make" do
       begin
@@ -26,7 +26,7 @@ module Prick::SubCommand
             clean = true
           end
 
-          builder = Prick::Build::Builder.new(conn, "schema", clean)
+          builder = Prick::Build::Builder.new(conn, "schema", clean, step: step)
 
           if schema 
             after_schemas = builder.pool.after_schema(schema)

data/lib/prick/subcommand/prick-run.rb

@@ -3,7 +3,7 @@
 require_relative '../builder/builder.rb'
 
 module Prick::SubCommand
-  def self.run(database, username, path, timer: nil, dump: nil, schema: nil)
+  def self.run(database, username, path, step: false, timer: nil, dump: nil, schema: nil)
 
     Timer.on! if timer
 
@@ -27,7 +27,7 @@ module Prick::SubCommand
           conn = Prick.state.connection
 
           # Parse
-          builder = Prick::Build::Builder.new(conn, path, single: true)
+          builder = Prick::Build::Builder.new(conn, path, single: true, load_pool: false, step: step)
 
           # Register if a build file is referenced and normalize path to
           # include build.yml of directories
@@ -43,9 +43,7 @@ module Prick::SubCommand
           # Read schema from build file if possible and decide if schema should
           # be dropped beforehand
           if is_build_file
-            pool = builder.pool
-            build_node = pool.all_nodes.first
-            if build_node.has_schema
+            if builder.root.has_schema
               !schema or Prick.error "Can't use --schema when doing a schema build"
               is_schema_rebuild = true
               schema = build_node.schema
@@ -67,6 +65,9 @@ module Prick::SubCommand
             end
           end
 
+          # Write back schema to builder
+          builder.root.schema = schema
+
           # Drop schema if needed
           if is_schema_rebuild
             conn.schema.drop schema, cascade: true 
@@ -78,26 +79,29 @@ module Prick::SubCommand
           end
         end
 
-        case dump
-          when :nodes; builder.nodes.reject { |node| node.is_a?(Build::BuildNode) }.map &:dump
-          when :allnodes; builder.nodes.map &:dump
-          when :batches; builder.dump
-          when nil;
+        if dump
+          builder.load_pool
+          case dump
+            when :nodes; builder.nodes.reject { |node| node.is_a?(Build::BuildNode) }.map &:dump
+            when :allnodes; builder.nodes.map &:dump
+            when :batches; builder.dump
+          else
+            Prick.error "Illegal dump type: #{dump.inspect}"
+          end
         else
-          Prick.error "Illegal dump type: #{dump.inspect}"
-        end && exit
-
-        Timer.time "Execute build object" do
-          builder.execute conn
+          Timer.time "Execute build object" do
+            builder.execute conn
+          end
         end
 
-      rescue Prick::Error => ex
-        $stderr.puts ex.message
-        exit 1
-
-      rescue ::Command::Error => ex
-        $stderr.puts ex.message
-        exit 1
+#     FIXME
+#     rescue Prick::Error => ex
+#       $stderr.puts ex.message
+#       exit 1
+#
+#     rescue ::Command::Error => ex
+#       $stderr.puts ex.message
+#       exit 1
 
       ensure
         super_conn&.terminate

data/lib/prick/subcommand/prick-snapshot.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require_relative '../builder/builder.rb'
+
+module Prick::SubCommand
+  def self.snapshot(
+      database, username, 
+      builddir: Prick.state.schema_dir)
+
+    conn = Prick.state.connection
+    builder = Prick::Build::Builder.new(conn, builddir)
+    pool = builder.pool
+    schemas = pool.refresh_schemas
+
+    # Clear snapshots table
+    conn.exec "delete from prick.snapshots"
+
+    # Fill it again
+    records = []
+    for schema in schemas
+      conn.schema.list_tables(schema).each { |table|
+        max_id = conn.value "select max(id) from #{schema}.#{table}"
+        records << { schema_name: schema, table_name: table, max_id: max_id}
+      }
+    end
+    conn.insert("prick", "snapshots", records)
+  end
+end
+

data/lib/prick/subcommand/subcommand.rb

@@ -1,4 +1,4 @@
-module Prick::Subcommand 
+module Prick::SubCommand 
 # def self.parse_username_database
 #   case username_database
 #     when nil

data/lib/prick/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Prick
-  VERSION = "0.34.0"
+  VERSION = "0.36.0"
 end

data/lib/prick.rb

@@ -53,6 +53,7 @@ require_relative 'prick/subcommand/prick-release.rb'
 require_relative 'prick/subcommand/prick-run.rb'
 require_relative 'prick/subcommand/prick-set.rb'
 require_relative 'prick/subcommand/prick-setup.rb'
+require_relative 'prick/subcommand/prick-snapshot.rb'
 require_relative 'prick/subcommand/prick-teardown.rb'
 
 module Prick

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prick
 version: !ruby/object:Gem::Version
-  version: 0.34.0
+  version: 0.36.0
 platform: ruby
 authors:
 - Claus Rasmussen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-28 00:00:00.000000000 Z
+date: 2024-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: semantic
@@ -233,6 +233,7 @@ files:
 - lib/prick/subcommand/prick-run.rb
 - lib/prick/subcommand/prick-set.rb
 - lib/prick/subcommand/prick-setup.rb
+- lib/prick/subcommand/prick-snapshot.rb
 - lib/prick/subcommand/prick-teardown.rb
 - lib/prick/subcommand/subcommand.rb
 - lib/prick/version.rb