petasos 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5b1ffed36c52ec275b881ac07ff08b6dda3d4446ba8c2ab0b74266698044b7e6
+  data.tar.gz: 3b42c94a62222b7f133c4fe0ed5caea15d1623bc49f1c953b07c9c7bf1fb64fe
+SHA512:
+  metadata.gz: 93023bc9d2cad8a5be3bf800683264b7ef20247b3824c2191a6e75dc4c0c0a311b2c595dec10d1377b8e1e3152f1d141f7d5d9a39ea1d02b7c535014dada2adb
+  data.tar.gz: 868e3587fc99dfd6a1cdc4b74d1ac5045d948f20b149bd7bb0294c72dfb2aa8d1afef0d6cd3a03ee1a11c02c4903b2fda2df818e076e336476126f34e7fdb1f6

data/bin/petasos

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'petasos'
+
+Petasos.new.run

data/lib/petasos/distributor.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "rake"
+require "yaml"
+require "petasos/node"
+
+class Petasos::Distributor
+  attr_reader :config
+
+  def initialize(config)
+    @config = config
+  end
+
+  def run
+    @nodes = {}
+    @config.each do |node|
+      @nodes[node["name"]] = Petasos::Node.new(node)
+    end
+
+    @manifests = {}
+    @nodes.each_pair do |node_name, node|
+      @manifests[node_name] = node.manifests
+    end
+
+    # {"petasos-node-a"=>
+    #     [{"name"=>"linux-laptop-source",
+    #       "imports"=>{},
+    #       "exports"=>
+    #        {"wow-ah"=>
+    #          {"path"=>"/home/justin/play/petasos/test/sandbox/node_a/location_a",
+    #           "canonical"=>true}}}],
+    #    "petasos-node-b"=>
+    #     [{"name"=>"linux-laptop-storage",
+    #       "imports"=>
+    #        {"wow-ah"=>
+    #          {"import_path"=>
+    #            "/home/justin/play/petasos/test/sandbox/node_b/location_a/data",
+    #           "backfill"=>true}},
+    #       "exports"=>{}}]}
+
+    @pools = Hash.new { |h, k|
+      h[k] = {
+        "import_paths" => [],
+        "backfill_import_paths" => [],
+        "canonical_exporters" => [],
+      }
+    }
+
+    @manifests.each_pair do |node_name, manifest_list|
+      manifest_list.each do |manifest|
+        manifest["imports"].each_pair do |pool_name, import_hash|
+          @pools[pool_name]["import_paths"] << [node_name, import_hash["import_path"]]
+          @pools[pool_name]["backfill_import_paths"] << [node_name, manifest["name"], import_hash["import_path"]] if import_hash["backfill"]
+        end
+        manifest["exports"].each_pair do |pool_name, export_hash|
+          @pools[pool_name]["canonical_exporters"] << [node_name, manifest["name"]] if export_hash["canonical"]
+        end
+      end
+    end
+
+    # {"wow-ah"=>
+    #     {"import_paths"=>
+    #       [["petasos-node-b",
+    #         "/home/justin/play/petasos/test/sandbox/node_b/location_a/data"]],
+    #      "backfill_import_paths"=>
+    #       [["petasos-node-b",
+    #         "linux-laptop-storage",
+    #         "/home/justin/play/petasos/test/sandbox/node_b/location_a/data"]],
+    #      "canonical_exporters"=>[["petasos-node-a", "linux-laptop-source"]]}}
+
+    # Process the exports files and return them as completed.
+    FileList.new(File.join(Dir.pwd, "**/exports_*")).each do |exports_file_path|
+      from_node_name = File.basename(File.dirname(exports_file_path))
+      from_node = find_node(from_node_name)
+      export_filename = File.basename(exports_file_path, ".*")
+      label, location_name, pool_name, datetime = export_filename.split("_")
+      export_paths = YAML.load_file(exports_file_path)
+      export_paths.each do |export_path|
+        @pools[pool_name]["import_paths"].each do |pool_storage|
+          to_node = find_node(pool_storage.first)
+          `scp #{from_node.host}:#{export_path}* #{to_node.host}:#{pool_storage.last}`
+        end
+      end
+      # mark it as completed
+      completed_export_file_path = File.join(Dir.pwd, "completed-#{File.basename(exports_file_path)}")
+      `mv #{exports_file_path} #{completed_export_file_path}`
+      # and then put it back where it came from
+      `scp #{completed_export_file_path} #{from_node.host}:#{from_node.path}`
+      `rm #{completed_export_file_path}`
+    end
+
+    # {"wow-ah"=>
+    #     {"import_paths"=>
+    #       [["petasos-node-b",
+    #         "/home/justin/play/petasos/test/sandbox/node_b/location_a/data"]],
+    #      "backfill_import_paths"=>
+    #       [["petasos-node-b",
+    #         "linux-laptop-storage",
+    #         "/home/justin/play/petasos/test/sandbox/node_b/location_a/data"]],
+    #      "canonical_exporters"=>[["petasos-node-a", "linux-laptop-source"]]}}
+
+    # Process the backfills.
+    # grab the seen files on the canonical exporters
+    @pools.each_pair do |pool_name, manifest_hash|
+      manifest_hash["canonical_exporters"].each do |canonical_exporter_details|
+        find_node(canonical_exporter_details.first).grab_seen_file_for_location(canonical_exporter_details.last, pool_name)
+      end
+
+      # grab the seen files on the backfill importers
+      manifest_hash["backfill_import_paths"].each do |backfill_importer_details|
+        find_node(backfill_importer_details.first).grab_seen_file_for_location(backfill_importer_details[1], pool_name)
+      end
+
+      # for each canonical exporter loop through the backfill lists, identify files that need moving and move them
+      manifest_hash["canonical_exporters"].each do |canonical_exporter_details|
+        exporter_seen_files = {}
+        exporter_file_list = YAML.load_file("seen_#{canonical_exporter_details.last}_#{pool_name}.yaml")
+        exporter_file_list.each { |f| exporter_seen_files[File.basename(f)] = f }
+        manifest_hash["backfill_import_paths"].each do |backfill_importer_details|
+          backfill_importer_files = {}
+          backfill_file_list = YAML.load_file("seen_#{backfill_importer_details[1]}_#{pool_name}.yaml")
+          backfill_file_list.each { |f| backfill_importer_files[File.basename(f)] = f }
+
+          exporter_seen_files.each_pair do |file_name, file_path|
+            unless backfill_importer_files[file_name]
+              from_node = find_node(canonical_exporter_details.first)
+              to_node = find_node(backfill_importer_details.first)
+              `scp #{from_node.host}:#{file_path} #{to_node.host}:#{backfill_importer_details.last}`
+            end
+          end
+        end
+      end
+      # clear the seen files locally.
+      `rm seen_*`
+    end
+  end
+
+  def find_node(node_name)
+    @nodes[node_name]
+  end
+end

data/lib/petasos/location.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "rake"
+require "yaml"
+
+class Petasos::Location
+  attr_reader :config
+
+  def initialize(config)
+    @config = config
+    initialize_all_seen_pool_files
+    update_manifest_file
+  end
+
+  def run
+    # delete exports file if completed file exists
+    FileList.new(File.join(Dir.pwd, "exports_#{@config["name"]}*.yaml")).each do |export_file_path|
+      completed_export_file_path = "completed-" + export_file_path
+      `rm #{export_file_path}` if File.file?(completed_export_file_path)
+    end
+
+    pools.each do |pool|
+      # get all filenames in this location that belong to this pool
+      current_files = current_pool_files(pool)
+
+      # get all filenames from the list of seen files
+      seen_pool_files = read_seen_pool_files(pool)
+
+      # identify which are new
+      new_files = current_files - seen_pool_files
+
+      # put a list of the new files where the cluster manager can find it
+      # if we are an exporter
+      if pool["export"]
+        create_file_export_list(pool, new_files.to_a) if new_files.length > 0
+      end
+
+      # this is where the "after_seen" hooks would run
+
+      # update list of seen files
+      update_seen_pool_files(pool, seen_pool_files + new_files)
+    end
+  end
+
+  def pools
+    config["pools"]
+  end
+
+  def update_manifest_file
+    # a list of pools and their import paths from locations
+    pool_imports = Hash.new { |h, k| h[k] = {} }
+    pool_exports = Hash.new { |h, k| h[k] = {} }
+    pools.each do |pool|
+      pool_import_path = pool["import_path"]
+      if pool_import_path
+        pool_imports[pool["name"]]["import_path"] = File.join(pool["path"], pool_import_path)
+        pool_imports[pool["name"]]["backfill"] = pool["backfill"] ? true : false
+      end
+      if pool["export"]
+        pool_exports[pool["name"]]["path"] = pool["path"]
+        pool_exports[pool["name"]]["canonical"] = pool["canonical"] ? true : false
+      end
+    end
+    manifest_hash = {
+      "name" => config["name"],
+      "imports" => pool_imports,
+      "exports" => pool_exports,
+    }
+    write_yaml("manifest_#{config["name"]}.yaml", manifest_hash)
+  end
+
+  def included_matchers(pool)
+    (pool["included_matchers"] || ["**/*.*"]).map { |fp| File.join(pool["path"], fp) }
+  end
+
+  def excluded_matchers(pool)
+    (pool["excluded_matchers"] || []).map { |fp| File.join(pool["path"], fp) }
+  end
+
+  def current_pool_files(pool)
+    FileList.new(included_matchers(pool)).exclude(excluded_matchers(pool))
+  end
+
+  def read_seen_pool_files(pool)
+    YAML.load_file(File.join(Dir.pwd, "seen_#{config["name"]}_#{pool["name"]}.yaml"))
+  end
+
+  def update_seen_pool_files(pool, file_paths)
+    yaml_path = File.join(Dir.pwd, "seen_#{config["name"]}_#{pool["name"]}.yaml")
+    write_yaml(yaml_path, file_paths)
+  end
+
+  def initialize_all_seen_pool_files
+    pools.each do |pool|
+      yaml_path = "seen_#{config["name"]}_#{pool["name"]}.yaml"
+      write_yaml(yaml_path, []) unless File.file?(yaml_path)
+    end
+  end
+
+  def clear_all_seen_pool_files
+    pools.each do |pool|
+      yaml_path = File.join(path, "seen_#{pool["name"]}.yaml")
+      write_yaml(yaml_path, [])
+    end
+  end
+
+  def create_file_export_list(pool, file_paths)
+    yaml_path = File.join(Dir.pwd, "exports_#{@config["name"]}_#{pool["name"]}_#{Time.now.strftime("%Y-%m-%d-%H:%M:%S")}.yaml")
+    write_yaml(yaml_path, file_paths)
+  end
+
+  def write_yaml(yaml_path, content)
+    File.open(yaml_path, "w") do |out|
+      YAML.dump(content, out)
+    end
+  end
+end

data/lib/petasos/node.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "rake"
+require "yaml"
+
+class Petasos::Node
+  attr_reader :config, :manifests
+
+  def initialize(config)
+    @config = config
+    @manifests = []
+    `mkdir -p #{config["name"]}`
+    grab_manifest_and_exports
+    parse_manifests
+  end
+
+  def host
+    @config["host"]
+  end
+
+  def path
+    @config["path"]
+  end
+
+  def grab_manifest_and_exports
+    `rsync #{config["host"]}:#{config["path"]}/manifest* #{config["name"]}/`
+    `rsync --ignore-missing-args --ignore-existing #{config["host"]}:#{config["path"]}/exports* #{config["name"]}/`
+  end
+
+  def grab_seen_file_for_location(location_name, pool_name)
+    `scp #{config["host"]}:#{config["path"]}/seen_#{location_name}_#{pool_name}.yaml .`
+  end
+
+  def parse_manifests
+    FileList.new("#{config["name"]}/manifest_*").each do |manifest_file_path|
+      @manifests << YAML.load_file(manifest_file_path)
+    end
+  end
+end

data/lib/petasos.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "rake"
+require "yaml"
+
+class Petasos
+  class Error < StandardError; end
+
+  def run
+    process_locations
+    process_distribution if File.file?(File.join(Dir.pwd, "petasos_distribution-config.yaml"))
+  end
+
+  def process_locations
+    # look for petasos_location-*.yaml files
+    # and pass each one to a petasos location manager
+    FileList.new("petasos_location-*.yaml").each do |location_file|
+      YAML.load_file(location_file).each do |location|
+        Petasos::Location.new(location).run
+      end
+    end
+  end
+
+  def process_distribution
+    # look for petasos_distribution-*.yaml files
+    # and pass each one to a petasos distribution
+    FileList.new("petasos_distribution-*.yaml").each do |distribution_file|
+      node_config = YAML.load_file(distribution_file)
+      Petasos::Distributor.new(node_config).run
+    end
+  end
+end
+
+require "petasos/location"
+require "petasos/node"
+require "petasos/distributor"

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: petasos
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Justin Myers
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-02-14 00:00:00.000000000 Z
+dependencies: []
+description: Petasos identifies new files and distributes them to where they belong
+email:
+- justin@tenmillionyears.org
+executables:
+- petasos
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/petasos
+- lib/petasos.rb
+- lib/petasos/distributor.rb
+- lib/petasos/location.rb
+- lib/petasos/node.rb
+homepage: https://github.com/JustinMyers/petasos
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key: 
+specification_version: 4
+summary: Petasos identifies new files and distributes them to where they belong
+test_files: []