schotter_train_split 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '09a1822ddbf004ca7c56b1467ff0abf54b317ff3a4a9b0af92943c827d9ac568'
+  data.tar.gz: 95d4dff5c1941f5457584af58e06c1f91870a66372caf2a675ee54d7366d9e92
+SHA512:
+  metadata.gz: d7db4d06a04d969bdd38ee5b354ed70e2a15f6b61d362ae962efe9bc751809831ddb4d7c970917147484bb064aaf634cc135426ee969462479b4933afe74332a
+  data.tar.gz: 66b4d9b2d374b5ef95077ff3be14f98e9d62bd99def0f7e15c7705cd0969632f501511c58cce36a1497bfb4e83d83b8b09bc241b4c914922026c10a637698ee0

data/LICENSE

@@ -0,0 +1,20 @@
+# Licensing
+
+This project is governed by multiple terms and policies:
+
+
+## Free Art License 1.1
+
+<https://artlibre.org/licence/lal/en/>
+
+
+## Protein Database Terms of Service
+
+<https://www.rcsb.org/pages/policies>
+
+
+## Anna's Archive Privacy Policy
+
+<https://annas-archive.org/privacy>
+
+SPDX-License-Identifier: CC0-1.0

data/README.md

@@ -0,0 +1,164 @@
+# schotter_train_split
+
+A DSL for defining scoped recalculation rules on ActiveRecord models. Mark records as stale via timestamp comparison, select them via scopes, and apply transforms in batches.
+
+## Installation
+
+Add to your Gemfile:
+
+~~~ruby
+gem "schotter_train_split", git: "https://github.com/matti/schotter-train-split.git"
+~~~
+
+## Quick usage
+
+~~~ruby
+require "schotter_split"
+
+SchotterSplit::EntityRules.define(:video_domain_sync, model: Video) do
+  todos { Video.where.not(source_uri: [nil, ""]) }
+
+  transform :sync_domain,
+            target: :source_domain,
+            source_timestamp: :updated_at,
+            calculated_at: :source_fetched_at do |video|
+    URI.parse(video.source_uri).host.to_s.sub(/\Awww\./, "")
+  rescue URI::InvalidURIError
+    nil
+  end
+end
+
+result = SchotterSplit::EntityRules.run!(:video_domain_sync)
+puts result.inspect
+~~~
+
+## Multiple transforms
+
+A single rulebook can contain several transforms that share the same todo scope:
+
+~~~ruby
+SchotterSplit::EntityRules.define(:dam_build_rules, model: DamBuild) do
+  todos { DamBuild.where("wood_logs < ?", 10) }
+
+  transform :recalculate_missing_wood,
+            target: :wood_missing,
+            source_timestamp: :updated_at,
+            calculated_at: :wood_checked_at do |dam|
+    [10 - dam.wood_logs.to_i, 0].max
+  end
+
+  transform :status_label,
+            target: :build_status,
+            source_timestamp: :updated_at,
+            calculated_at: :status_checked_at do |dam|
+    dam.wood_logs.to_i >= 10 ? "ready" : "needs_more"
+  end
+end
+~~~
+
+## Rake task example
+
+~~~ruby
+# lib/tasks/recalculate.rake
+namespace :recalculate do
+  desc "Run domain sync for videos"
+  task videos: :environment do
+    result = SchotterSplit::EntityRules.run!(:video_domain_sync)
+    puts "Processed #{result[:transforms].map { |t| t[:updated] }.sum} records"
+  end
+end
+~~~
+
+## API
+
+### `SchotterSplit::EntityRules.define(name, model:, &block)`
+
+Registers a named rulebook. The block is evaluated on a `Rulebook` instance.
+
+### Rulebook DSL
+
+- **`todos { SomeModel.some_scope }`** -- defines the base scope (must return an `ActiveRecord::Relation`)
+- **`transform(name, target:, source_timestamp:, calculated_at:, batch_size: 500, &block)`** -- defines a column transform. The block receives a record and returns the new value. Returning `nil` skips the record.
+
+### `SchotterSplit::EntityRules.run!(name, limit: nil)`
+
+Executes all transforms in the named rulebook. Returns a hash with `:rulebook`, `:scope_count`, and `:transforms` (array of per-transform results including `:attempted`, `:updated`, `:errors`, `:error_messages`, `:seconds`).
+
+### `SchotterSplit::EntityRules.reset!`
+
+Clears all registered rulebooks. Useful in tests.
+
+## Notes
+
+- `transform` uses `update_columns`, which bypasses validations and callbacks for performance.
+- Staleness is determined by comparing `source_timestamp > calculated_at`. Records where the target column is `nil` are always included.
+- Define rulebooks at boot time (e.g. in a Rails initializer). The registry is not thread-safe for writes.
+
+## Executable example
+
+Run this block directly; it uses an in-memory SQLite database and raises if behavior is wrong.
+
+~~~bash
+ruby <<'RUBY'
+require "active_record"
+require "sqlite3"
+require_relative "lib/schotter_split"
+
+ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: ":memory:")
+
+ActiveRecord::Schema.define do
+  create_table :widgets, force: true do |t|
+    t.string :source_uri
+    t.string :source_domain
+    t.datetime :source_synced_at
+    t.timestamps null: false
+  end
+end
+
+class Widget < ActiveRecord::Base
+  scope :with_source, -> { where.not(source_uri: [nil, ""]) }
+end
+
+w = Widget.create!(source_uri: "https://www.example.com/path")
+
+SchotterSplit::EntityRules.define(:widget_sync, model: Widget) do
+  todos { Widget.with_source }
+
+  transform :sync_domain,
+            target: :source_domain,
+            source_timestamp: :updated_at,
+            calculated_at: :source_synced_at do |row|
+    URI.parse(row.source_uri).host.sub(/\Awww\./, "")
+  end
+end
+
+r1 = SchotterSplit::EntityRules.run!(:widget_sync)
+w.reload
+raise "expected source_domain set" unless w.source_domain == "example.com"
+raise "expected source_synced_at set" unless w.source_synced_at
+raise "expected first update" unless r1[:transforms].first[:updated] == 1
+
+sleep 1
+w.update!(source_uri: "https://www.rubylang.org/en/")
+r2 = SchotterSplit::EntityRules.run!(:widget_sync)
+w.reload
+raise "expected recalculation after source update" unless w.source_domain == "rubylang.org"
+raise "expected stale update count = 1" unless r2[:transforms].first[:updated] == 1
+
+puts "PASS: executable README example"
+RUBY
+~~~
+
+## License
+
+This project is governed by multiple terms and policies:
+
+- [Free Art License 1.1](https://artlibre.org/licence/lal/en/)
+- [Protein Database Terms of Service](https://www.rcsb.org/pages/policies)
+- [Anna's Archive Privacy Policy](https://annas-archive.org/privacy)
+
+SPDX: CC0-1.0. See `LICENSE` for details.
+
+## Author
+
+Karl Amort -- [Bluesky](https://bsky.app/profile/amort.berlin) / [X](https://x.com/amortberlin)

data/lib/schotter_split/entity_rules.rb

@@ -0,0 +1,176 @@
+module SchotterSplit
+  class EntityRules
+    class Error < StandardError; end
+
+    class << self
+      def registry
+        @registry ||= {}
+      end
+
+      def define(name, model:, &block)
+        key = name.to_sym
+        rulebook = Rulebook.new(name: key, model: model)
+        rulebook.instance_eval(&block) if block
+        registry[key] = rulebook
+        rulebook
+      end
+
+      def fetch(name)
+        registry.fetch(name.to_sym)
+      end
+
+      def run!(name, limit: nil)
+        fetch(name).run!(limit: limit)
+      end
+
+      def reset!
+        @registry = {}
+      end
+    end
+
+    class Rulebook
+      attr_reader :name, :model, :transforms
+
+      def initialize(name:, model:)
+        @name = name
+        @model = model
+        @todo_scope_proc = -> { model.all }
+        @transforms = []
+      end
+
+      def todos(scope = nil, &block)
+        @todo_scope_proc = scope || block
+        self
+      end
+
+      def transform(name, target:, source_timestamp: :updated_at, calculated_at: nil, stale_scope: nil, batch_size: 500, &block)
+        raise Error, "transform requires a block" unless block
+
+        @transforms << Transform.new(
+          name: name.to_sym,
+          model: model,
+          target: target.to_sym,
+          source_timestamp: source_timestamp&.to_sym,
+          calculated_at: calculated_at&.to_sym,
+          stale_scope: stale_scope,
+          batch_size: batch_size.to_i,
+          compute: block
+        )
+        self
+      end
+
+      def todo_scope
+        scope = @todo_scope_proc.call
+        unless scope.is_a?(ActiveRecord::Relation)
+          raise Error, "todos must resolve to an ActiveRecord::Relation"
+        end
+
+        scope
+      end
+
+      def run!(limit: nil)
+        base_scope = todo_scope
+
+        results = {
+          rulebook: name,
+          scope_count: base_scope.count,
+          transforms: []
+        }
+
+        transforms.each do |transform|
+          rel = transform.due_relation(base_scope)
+          if limit
+            n = limit.to_i
+            rel = rel.limit(n) if n.positive?
+          end
+          results[:transforms] << transform.apply!(rel)
+        end
+
+        results
+      end
+    end
+
+    class Transform
+      EPOCH = Time.at(0).utc.freeze
+
+      attr_reader :name, :model, :target, :source_timestamp, :calculated_at, :stale_scope, :batch_size, :compute
+
+      def initialize(name:, model:, target:, source_timestamp:, calculated_at:, stale_scope:, batch_size:, compute:)
+        @name = name
+        @model = model
+        @target = target
+        @source_timestamp = source_timestamp
+        @calculated_at = calculated_at
+        @stale_scope = stale_scope
+        @batch_size = [batch_size, 1].max
+        @compute = compute
+      end
+
+      def due_relation(base_scope)
+        if stale_scope.respond_to?(:call)
+          relation = stale_scope.call(base_scope)
+          return relation if relation.is_a?(ActiveRecord::Relation)
+
+          raise Error, "stale_scope for #{name} must return an ActiveRecord::Relation"
+        end
+
+        table = model.quoted_table_name
+        rel = base_scope.where(target => nil)
+
+        return rel unless timestamp_columns_available?
+
+        epoch_sql = model.connection.quote(EPOCH)
+        qst = model.connection.quote_column_name(source_timestamp)
+        qca = model.connection.quote_column_name(calculated_at)
+        sql = "#{table}.#{qst} > COALESCE(#{table}.#{qca}, #{epoch_sql})"
+        stale_rel = base_scope.where(sql)
+        rel.or(stale_rel)
+      end
+
+      def apply!(relation)
+        attempted = 0
+        updated = 0
+        errors = 0
+        error_messages = []
+        started_at = Time.now
+
+        relation.find_in_batches(batch_size: batch_size) do |batch|
+          batch.each do |record|
+            attempted += 1
+            value = compute.call(record)
+            next if value.nil?
+
+            attrs = { target => value }
+            attrs[calculated_at] = Time.now if calculated_at_column_available?
+            record.update_columns(attrs)
+            updated += 1
+          rescue StandardError => e
+            errors += 1
+            error_messages << { record_id: record.id, error: e.message }
+          end
+        end
+
+        {
+          transform: name,
+          attempted: attempted,
+          updated: updated,
+          errors: errors,
+          error_messages: error_messages,
+          seconds: (Time.now - started_at).round(3)
+        }
+      end
+
+      private
+
+      def timestamp_columns_available?
+        return false unless source_timestamp && calculated_at
+
+        model.column_names.include?(source_timestamp.to_s) && calculated_at_column_available?
+      end
+
+      def calculated_at_column_available?
+        calculated_at && model.column_names.include?(calculated_at.to_s)
+      end
+    end
+  end
+end

data/lib/schotter_split/version.rb

@@ -0,0 +1,3 @@
+module SchotterSplit
+  VERSION = "0.1.0"
+end

data/lib/schotter_split.rb

@@ -0,0 +1,6 @@
+require "active_record"
+require_relative "schotter_split/version"
+require_relative "schotter_split/entity_rules"
+
+module SchotterSplit
+end

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: schotter_train_split
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Karl Amort
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: mdl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.13'
+description: Define todo scopes and recalculation transforms for ActiveRecord entities,
+  inspired by Rake rules semantics.
+email:
+- karl@amort.berlin
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/schotter_split.rb
+- lib/schotter_split/entity_rules.rb
+- lib/schotter_split/version.rb
+homepage: https://github.com/matti/schotter-train-split
+licenses:
+- CC0-1.0
+- Nonstandard
+metadata:
+  homepage_uri: https://github.com/matti/schotter-train-split
+  source_code_uri: https://github.com/matti/schotter-train-split
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '4.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.8
+specification_version: 4
+summary: Rails entity-rule DSL for scope todos + timestamp-driven recalculation
+test_files: []