openlayer 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c522a801c7673ccfe15d08aa8dffa01df36e9247cdd49181fb2f4f0a09b6ecc
-  data.tar.gz: ce80b6e2d2333d8f9118bac2477835c0b2046994726513547cc82b61c3d25d61
+  metadata.gz: 9124105239930d01d444266033af79f2e1d2a7e6048b587e13dc6cfa1d00a0c3
+  data.tar.gz: 869c8e6a79e351d2b5047272558e4b33fb7e794276731cfbe4e246410804d57b
 SHA512:
-  metadata.gz: a082fa2d42ce9fa5d1c13b8f883f675fc1235e4dcbc9699ba26e4bf06f7aff7fecee96295393dc9de650f8b388621b2fc939a20d38eb912943404a79f8595dd2
-  data.tar.gz: cdb4dc6acbd4d643ed22a2b3655b949bdd4b99b34c35aaf798583e59175abbb40ab234ec4ab4a1809015457ab3faad820bde8dd0abb2bcad9b3fa0ac30b4dc61
+  metadata.gz: aaa1a3ec31f4dd6ef75ba1f3acd5d50cb49318ac899dd3ec0442ddbe332abebe1ff6c39b042f4cc9ec66b05bd59790baf4bcf2ebe26f601b36d823e228940f0a
+  data.tar.gz: 7d76c46e8c8b5d468a16421dc9d859c6768cdb1981afcd72b73cbfcfa307d3bdce1cd37f16066f4b46d7ab248ab1aef2cfe6f79ae20cee2ec0a72fc71fe667ed

data/README.md

@@ -13,6 +13,7 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
+### Inference Monitoring Mode
 ```ruby
 client = Openlayer::Client.new(api_key: "YOUR_OPENLAYER_API_KEY")
 inference_pipeline = client.inference_pipeline("YOUR_INFERENCE_PIPELINE_ID")
@@ -24,6 +25,39 @@ inference_pipeline.stream_data(
 )
 ```
 
+
+
+### Projects
+#### Load Project
+Loads an existing project from the Openlayer platform.
+
+```ruby
+client = Openlayer::Client.new(api_key: "YOUR_OPENLAYER_API_KEY")
+project = client.load_project(name: "your_project_name")
+```
+
+#### Development Mode for CI/CD
+```ruby
+client = Openlayer::Client.new(api_key: "YOUR_OPENLAYER_API_KEY")
+project = client.load_project(name: "your_project_name")
+
+
+project.add_dataset(file_path: "DATASET_FILE_PATH", dataset_config_file_path: "DATASET_CONFIG_FILE_PATH")
+project.add_model(model_config_file_path: "MODEL_CONFIG_PATH")
+project.commit(message: "This is a commit message")
+version = project.push
+
+version.wait_for_completion(timeout: 500)
+version.print_test_report
+
+if version.failing_test_count > 0:
+    puts "Failing pipeline due to failing goals."
+    exit(1)
+```
+
+- Check the example in `examples/staging` for what the YAMLs look like.
+- Check the [Dataset Config](https://docs.openlayer.com/how-to-guides/write-dataset-configs/llm-dataset-config) and the [Model Config](https://docs.openlayer.com/how-to-guides/write-model-configs/llm-config) docs on Openlayer to see additional attributes for the configs.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/examples/staging/commit.yaml

@@ -0,0 +1,2 @@
+date: Sat Feb 25 14:24:18 2023
+message: Initial commit!

data/examples/staging/model/model_config.yaml

@@ -0,0 +1,12 @@
+architectureType: llm
+inputVariableNames:
+  - input
+metadata: {}
+model: gpt-3.5-turbo
+modelParameters:
+  temperature: 0.7
+modelProvider: OpenAI
+modelType: api
+name: AI assistant
+prompt: 
+  - {"role": "user", "content": "{{ input }}"}

data/examples/staging/validation/dataset_config.yaml

@@ -0,0 +1,12 @@
+columnNames:
+  - input
+  - output
+  - ground_truth
+groundTruthColumnName: ground_truth
+inputVariableNames:
+  - input
+label: validation
+language: en
+metadata: {}
+outputColumnName: output
+sep: ","

data/lib/openlayer/client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "faraday"
+require "faraday/multipart"
 
 module Openlayer
   class Client
@@ -14,6 +15,25 @@ module Openlayer
       @stubs = stubs
     end
 
+    def inference_pipeline(inference_pipeline_id)
+      InferencePipeline.new(self, inference_pipeline_id)
+    end
+
+    def load_project(name:)
+      Project.from_response self, connection.get("projects", project_search_params(name: name))
+    end
+
+    def presigned_url(object_name: nil, workspace_id: nil)
+      handle_response connection.post(
+        "storage/presigned-url?objectName=staging", presigned_url_params(object_name: object_name,
+                                                                         workspace_id: workspace_id)
+      )
+    end
+
+    def load_project_version(id:, payload: {})
+      ProjectVersion.from_response self, connection.post("projects/#{id}/versions", payload)
+    end
+
     def connection
       @connection ||= Faraday.new do |conn|
         conn.url_prefix = BASE_URL
@@ -24,8 +44,34 @@ module Openlayer
       end
     end
 
-    def inference_pipeline(inference_pipeline_id)
-      InferencePipeline.new(self, inference_pipeline_id)
+    private
+
+    def project_search_params(name:)
+      { version: "0.1.0a25", name: name }
+    end
+
+    def presigned_url_params(object_name: nil, workspace_id: nil)
+      { "objectName": object_name, "workspaceId": workspace_id, version: "0.1.0a25" }.compact
+    end
+
+    def handle_response(response)
+      message = response.body["error"]
+      case response.status
+      when 200
+        response.body
+      when 401
+        raise Error, message
+      when 404
+        raise Error, message
+      when 422
+        raise Error, message
+      when 500
+        raise Error, message
+      when 200..299
+        response.body
+      else
+        raise Error, message
+      end
     end
   end
 end

data/lib/openlayer/object.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "ostruct"
+module Openlayer
+  class Object
+    def initialize(attributes)
+      @attributes = OpenStruct.new(attributes)
+    end
+
+    def method_missing(method, *args, &block)
+      method = snake_to_camel(method.to_s)
+      attribute = @attributes.send(method, *args, &block)
+      attribute.is_a?(Hash) ? Object.new(attribute) : attribute
+    end
+
+    def respond_to_missing?(_method, _include_private = false)
+      true
+    end
+
+    private
+
+    def snake_to_camel(snake_str)
+      snake_str.split("_").each_with_index.map do |part, index|
+        index.zero? ? part : part.capitalize
+      end.join
+    end
+  end
+end

data/lib/openlayer/objects/project.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "find"
+require "yaml"
+require "pry"
+
+module Openlayer
+  class Project < Object
+    class Error < StandardError; end
+    class NotFoundError < StandardError; end
+    class TarFileNotFoundError < StandardError; end
+
+    class CommitLengthError < StandardError
+      def message
+        "Commit message must be between 1 and 140 characters"
+      end
+    end
+
+    attr_reader :client, :workspace_id,
+                :data_tarfile_path, :s3_presigned_body,
+                :commit_message, :project_path
+
+    REQUIRED_TARFILE_STRUCTURE = [
+      "staging/commit.yaml",
+      "staging/validation/dataset_config.yaml",
+      "staging/validation/dataset.csv",
+      "staging/model/model_config.yaml"
+    ].freeze
+
+    COMMIT_LENGTH = (1..140).freeze
+
+    def self.from_response(client, response)
+      attributes_first_project = handle_response(response)&.dig("items")&.first
+      raise NotFoundError if attributes_first_project.nil?
+
+      new(client, attributes_first_project)
+    end
+
+    def initialize(client, attributes)
+      @client = client
+      super(attributes)
+      @data_tarfile_path = "#{Dir.home}/.openlayer/#{id}_staging.tar"
+      @project_path = "#{Dir.home}/.openlayer/#{id}"
+      init_staging_directories
+    end
+
+    def add_dataset(file_path:, dataset_config: nil, dataset_config_file_path: nil)
+      raise ArgumentError, "dataset config is required" if dataset_config.nil? && dataset_config_file_path.nil?
+
+      copy_file(file_path, "staging/validation/dataset.csv")
+      copy_or_create_config(dataset_config,
+                            dataset_config_file_path,
+                            "staging/validation/dataset_config.yaml")
+    end
+
+    def add_model(model_config: nil, model_config_file_path: nil)
+      raise ArgumentError, "model config is required" if model_config.nil? && model_config_file_path.nil?
+
+      copy_or_create_config(model_config,
+                            model_config_file_path,
+                            "staging/model/model_config.yaml")
+    end
+
+    def status
+      puts "Staging Area:"
+      Find.find(File.join(project_path, "/staging")) do |path|
+        puts path if File.file?(path)
+      end
+    end
+
+    def commit(message)
+      raise CommitLengthError unless COMMIT_LENGTH.include? message.length
+
+      @commit_message = message
+      commit_hash = {
+        "date": DateTime.now.strftime("%a %b %d %H:%M:%S %Y"),
+        "message": message
+      }
+      write_hash_to_file(commit_hash, "staging/commit.yaml")
+    end
+
+    def push
+      create_and_validate_tar_file
+      s3_client = create_s3_client
+      s3_client.post(s3_payload)
+      client.load_project_version(id: id, payload: version_payload)
+    end
+
+    private
+
+    def init_staging_directories
+      FileUtils.mkdir_p(File.join(project_path, "staging"))
+      FileUtils.mkdir_p(File.join(project_path, "staging/validation"))
+      FileUtils.mkdir_p(File.join(project_path, "staging/model"))
+    end
+
+    def copy_or_create_config(config_hash, config_file_path, destination)
+      if config_file_path.nil?
+        write_hash_to_file(config_hash, destination)
+      else
+        copy_file(config_file_path, destination)
+      end
+    end
+
+    def copy_file(file_path, destination)
+      FileUtils.copy(file_path, File.join(project_path, destination))
+    end
+
+    def write_hash_to_file(hash, destination)
+      File.open(File.join(project_path, destination), "w") do |file|
+        file.write hash.to_yaml
+      end
+    end
+
+    def create_s3_client
+      @s3_presigned_body = client.presigned_url(workspace_id: workspace_id, object_name: "staging")
+      S3PresignedClient.new(s3_presigned_body)
+    end
+
+    def create_and_validate_tar_file
+      TarFileHelper.create_tar_from_folders([File.join(project_path, "staging")], data_tarfile_path)
+      validate_tarfile(data_tarfile_path)
+    end
+
+    def validate_tarfile(file_path)
+      raise TarFileNotFoundError unless File.exist?(file_path)
+
+      TarFileHelper.validate_structure(file_path, REQUIRED_TARFILE_STRUCTURE)
+    end
+
+    def s3_payload
+      {
+        "file" => Faraday::Multipart::FilePart.new(data_tarfile_path, "application/x-tar")
+      }
+    end
+
+    def version_payload
+      {
+        "storageUri": s3_presigned_body["storageUri"],
+        "commit": { "message": commit_message }
+      }
+    end
+
+    def self.handle_response(response)
+      message = response.body["error"]
+      case response.status
+      when 200
+        response.body
+      when 401
+        raise Error, message
+      when 404
+        raise Error, message
+      when 422
+        raise Error, message
+      when 500
+        raise Error, message
+      when 200..299
+        response.body
+      else
+        raise Error, message
+      end
+    end
+
+    def handle_response(response)
+      message = response.body["error"]
+      case response.status
+      when 200
+        response.body
+      when 401
+        raise Error, message
+      when 404
+        raise Error, message
+      when 422
+        raise Error, message
+      when 500
+        raise Error, message
+      when 200..299
+        response.body
+      else
+        raise Error, message
+      end
+    end
+  end
+end

data/lib/openlayer/objects/project_version.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Openlayer
+  class ProjectVersion < Object
+    attr_reader :client, :project_version_id, :all_attributes_hash
+
+    def self.from_response(client, response)
+      attributes = handle_response(response)
+      new(client, attributes)
+    end
+
+    def initialize(client, attributes)
+      @client = client
+      super(attributes)
+      @project_version_id = commit.projectVersionId
+    end
+
+    def print_status_report
+      puts "Status: #{status}"
+      puts "Status Message: #{status_message}" unless status_message.nil?
+    end
+
+    def print_test_report
+      puts "Failing Test Count: #{failing_test_count}"
+      puts "Passing Test Count: #{passing_test_count}"
+      puts "Total Test Count: #{total_test_count}"
+    end
+
+    def to_h
+      @attributes.to_h
+    end
+
+    def wait_for_completion(timeout: 300, interval: 10)
+      start_time = Time.now
+      while Time.now - start_time < timeout
+        refresh
+        break if status == "completed" || status == "failed"
+
+        sleep interval
+      end
+    end
+
+    def refresh
+      @attributes = OpenStruct.new handle_response client.connection.get("versions/#{project_version_id}")
+    end
+
+    private
+
+    def self.handle_response(response)
+      message = response.body["error"]
+      case response.status
+      when 200
+        response.body
+      when 401
+        raise Error, message
+      when 404
+        raise Error, message
+      when 422
+        raise Error, message
+      when 500
+        raise Error, message
+      when 200..299
+        response.body
+      else
+        raise Error, message
+      end
+    end
+
+    def handle_response(response)
+      message = response.body["error"]
+      case response.status
+      when 200
+        response.body
+      when 401
+        raise Error, message
+      when 404
+        raise Error, message
+      when 422
+        raise Error, message
+      when 500
+        raise Error, message
+      when 200..299
+        response.body
+      else
+        raise Error, message
+      end
+    end
+  end
+end

data/lib/openlayer/services/s3_presigned_client.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Openlayer
+  class S3PresignedClient
+    attr_reader :presigned_url, :payload_fields, :s3_conn
+
+    def initialize(presigned_url)
+      @presigned_url = presigned_url
+      @payload_fields = presigned_url["fields"]
+      @s3_conn = Faraday.new(url: presigned_url["url"]) do |f|
+        f.request :multipart
+        f.adapter :net_http
+      end
+    end
+
+    def post(data)
+      handle_response @s3_conn.post("/", merge_fields(data))
+    end
+
+    private
+
+    def merge_fields(data)
+      payload_fields.merge!(data)
+    end
+
+    def handle_response(response)
+      message = response.body["error"]
+      case response.status
+      when 200
+        return response
+      when 401
+        raise Error, message
+      when 404
+        raise Error, message
+      when 422
+        raise Error, message
+      when 500
+        raise Error, message
+      when 200..299
+        return response
+      else
+        raise Error, message
+      end
+      response
+    end
+  end
+end

data/lib/openlayer/services/tar_file_helper.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "rubygems/package"
+require "fileutils"
+require "zlib"
+require "logger"
+
+module Openlayer
+  class TarFileHelper
+    LOGGER = Logger.new($stdout)
+
+    def self.create_tar_from_folders(folders, tar_file_name)
+      File.open(tar_file_name, "wb") do |file|
+        Gem::Package::TarWriter.new(Zlib::GzipWriter.open(file)) do |tar|
+          folders.each do |folder|
+            base_name = File.basename(folder)
+            add_directory_to_tar(tar, folder, base_name)
+          end
+        end
+      end
+    end
+
+    def self.validate_structure(file_path, required_structure)
+      tarfile_structure = []
+      begin
+        Gem::Package::TarReader.new(Zlib::GzipReader.open(file_path)).each do |entry|
+          tarfile_structure << entry.full_name
+        end
+      rescue Zlib::GzipFile::Error => e
+        stacktrace = e.backtrace.join("\n")
+        LOGGER.error("ERROR: #{e.message}\n #{stacktrace}")
+        return nil
+      end
+
+      required_structure.each do |required_file|
+        raise Error, "Missing file: #{required_file}" unless tarfile_structure.include?(required_file)
+      end
+    end
+
+    def self.add_directory_to_tar(tar, folder, base_name)
+      Dir[File.join(folder, "**/*")].each do |file|
+        mode = File.stat(file).mode
+        relative_file = File.join(base_name, file.sub("#{folder}/", ""))
+
+        if File.directory?(file)
+          tar.mkdir(relative_file, mode)
+        else
+          tar.add_file_simple(relative_file, mode, File.size(file)) do |io|
+            File.open(file, "rb") { |f| io.write(f.read) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/openlayer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Openlayer
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/openlayer.rb

@@ -5,7 +5,12 @@ require_relative "openlayer/version"
 module Openlayer
   autoload :Client, "openlayer/client"
   autoload :Error, "openlayer/error"
+  autoload :Object, "openlayer/object"
   autoload :InferencePipeline, "openlayer/inference_pipeline"
+  autoload :S3PresignedClient, "openlayer/services/s3_presigned_client"
+  autoload :TarFileHelper, "openlayer/services/tar_file_helper"
 
-  # Your code goes here...
+  # Classes used to return a nicer object wrapping the response data
+  autoload :Project, "openlayer/objects/project"
+  autoload :ProjectVersion, "openlayer/objects/project_version"
 end

data/openlayer.gemspec

@@ -34,7 +34,11 @@ Gem::Specification.new do |spec|
 
   # Uncomment to register a new dependency of your gem
   # spec.add_dependency "example-gem", "~> 1.0"
+  spec.add_dependency "date"
   spec.add_dependency "faraday", "~> 2.9.0"
+  spec.add_dependency "faraday-multipart"
+  spec.add_dependency "yaml"
+  spec.add_dependency "zlib"
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: openlayer
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Jose Fernando Davila
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-01-19 00:00:00.000000000 Z
+date: 2024-03-21 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: date
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -24,6 +38,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.9.0
+- !ruby/object:Gem::Dependency
+  name: faraday-multipart
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yaml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: zlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Openlayer api wrapper
 email:
 - davila.jose23@gmail.com
@@ -37,10 +93,19 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- examples/staging/.DS_Store
+- examples/staging/commit.yaml
+- examples/staging/model/model_config.yaml
+- examples/staging/validation/dataset_config.yaml
 - lib/openlayer.rb
 - lib/openlayer/client.rb
 - lib/openlayer/error.rb
 - lib/openlayer/inference_pipeline.rb
+- lib/openlayer/object.rb
+- lib/openlayer/objects/project.rb
+- lib/openlayer/objects/project_version.rb
+- lib/openlayer/services/s3_presigned_client.rb
+- lib/openlayer/services/tar_file_helper.rb
 - lib/openlayer/version.rb
 - openlayer.gemspec
 - sig/openlayer.rbs
@@ -65,7 +130,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: Openlayer api wrapper