dorian 2.0.2 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60015bb962654dc712a54a08aedaf13c7727ebca8ae136c031c4d595c29daff1
-  data.tar.gz: 3af0aba2481e360390435c7dd9d75d4d2201705b5ea3503c165fb4e82b50d6c7
+  metadata.gz: bd057371121d824ad1a6172b189916820b5e5544f9cabd95db4fd7f51a2bd5fa
+  data.tar.gz: 244af12257d30dc4c617b8443a4a02d2c8ecc0fb7209f7b27dee7525ee43b35f
 SHA512:
-  metadata.gz: 5630497aed0f3498f6e442aceeb47d138334a37049a6b1fbd6fdb61b722257d98a819876e22fbcd01a826ae144f5bda966503179140c8869de3694cd0597bfd6
-  data.tar.gz: a075cd1be6708d17d6d47733e7cadb8273fe3060ecda00610b8523a2c19cba7e5b84a6b564736fca7a15483394f187d539c422c369c56aaa148f0411836c1982
+  metadata.gz: a6e82fef1ff6f595dfecba39e6f14ad19bf502f38b8c4af6d5e261eb69cbd3461119fbbb526089cddea881a00cb26f3b11706b34969485d2e2ad61bce6829c27
+  data.tar.gz: a3f12ae1d9d88cb6f28b975f5b59ca11ed37baa612c3e8b6c58009bc639f59084ed1af3cb2f400ed94bfc38ed8a51b7ef0c6b93aa7a1ad8118a52c8e7b7cec41

data/VERSION

@@ -1 +1 @@
-2.0.2
+2.2.0

data/lib/dorian/bin.rb

@@ -5,9 +5,12 @@ require "dorian/arguments"
 require "dorian/eval"
 require "dorian/progress"
 require "dorian/to_struct"
+require "git"
 require "json"
-require "yaml"
+require "net/http"
 require "parallel"
+require "uri"
+require "yaml"
 
 class Dorian
   class Bin
@@ -21,10 +24,12 @@ class Dorian
       "jsonl" => :jsonl,
       "raw" => :raw,
       "yaml" => :yaml,
-      "yml" => :yaml
+      "yml" => :yaml,
+      "yamll" => :yamll,
+      "ymll" => :yamll
     }.freeze
 
-    attr_reader :parsed, :command, :arguments
+    attr_reader :parsed, :command, :arguments, :ruby, :ruby_before, :ruby_after
 
     def initialize
       @parsed =
@@ -47,6 +52,15 @@ class Dorian
           parallel: {
             alias: :p
           },
+          parallel_type: {
+            alias: :pt,
+            type: :string,
+            default: :processes
+          },
+          n: {
+            type: :integer,
+            default: 100
+          },
           rails: {
             alias: :r
           },
@@ -73,7 +87,8 @@ class Dorian
           progress: :boolean,
           headers: :boolean,
           progress_format: {
-            alias: :pf
+            alias: :pf,
+            type: :string
           },
           pretty: {
             default: true
@@ -87,7 +102,9 @@ class Dorian
           }
         )
 
-      @command, *@arguments = parsed.arguments
+      @arguments = parsed.arguments
+      @command = arguments.first
+      @ruby = nil
     end
 
     def self.run(...)
@@ -99,12 +116,71 @@ class Dorian
       abort VERSION if version?
 
       case command&.to_sym
-      when :read, nil
-        command_read
       when :each
+        arguments.delete("each")
+        @command = :each
+        @ruby = arguments.delete_at(0)
         command_each
+      when :all
+        arguments.delete("all")
+        @command = :all
+        @ruby = arguments.delete_at(0)
+        command_all
+      when :before
+        arguments.delete("before")
+        @command = :before
+        @ruby = arguments.delete_at(0)
+        command_before
+      when :after
+        arguments.delete("after")
+        @command = :after
+        @ruby = arguments.delete_at(0)
+        command_after
+      when :between
+        arguments.delete("between")
+        @command = :between
+        @ruby_after = arguments.delete_at(0)
+        @ruby_before = arguments.delete_at(0)
+        command_between
+      when :select
+        arguments.delete("select")
+        @command = :select
+        @ruby = arguments.delete_at(0)
+        command_select
+      when :reject
+        arguments.delete("reject")
+        @command = :reject
+        @ruby = arguments.delete_at(0)
+        command_reject
+      when :tally
+        arguments.delete("tally")
+        @command = :tally
+        @ruby = arguments.delete_at(0)
+        command_tally
+      when :anonymize
+        arguments.delete("anonymize")
+        @command = :anonymize
+        command_anonymize
+      when :append
+        arguments.delete("append")
+        @command = :append
+        command_append
+      when :prepend
+        arguments.delete("prepend")
+        @command = :prepend
+        command_prepend
+      when :chat
+        arguments.delete("chat")
+        @command = :chat
+        command_chat
+      when :commit
+        arguments.delete("commit")
+        @command = :commit
+        command_commit
       else
-        abort "#{command} not supported"
+        arguments.delete("read")
+        @command = :read
+        command_read
       end
     end
 
@@ -112,19 +188,154 @@ class Dorian
       parsed.files
     end
 
+    def command_chat
+      puts completion(
+             token: token(".chat"),
+             model: "gpt-4o",
+             messages: [{ role: :user, content: everything.join("\n") }]
+           )
+    end
+
+    def command_commit
+      system_prompt = "simple, clear, short, lowercase commit message"
+      prompt_1 = "for the following diff:"
+      prompt_2 = "for the following git status:"
+      prompt_3 = "for the following comment:"
+
+      content_1 = short(`git diff --staged`)
+      content_2 = short(`git status`)
+      content_3 = short(arguments.join("\n"))
+
+      abort "no staged files" if content_1.empty?
+
+      messages = [
+        { role: :system, content: system_prompt },
+        { role: :system, content: prompt_1 },
+        { role: :user, content: content_1 },
+        { role: :system, content: prompt_2 },
+        { role: :user, content: content_2 },
+        { role: :system, content: prompt_3 },
+        { role: :user, content: content_3 }
+      ]
+
+      message = completion(token: token(".commit"), model: "gpt-4o", messages: messages)
+
+      Git.open(".").commit(message)
+
+      puts message
+    end
+
     def command_read
       each(stdin_files + files) do |input|
         outputs(reads(File.read(input)), file: input)
       end
 
-      each(stdin_arguments) { |input| outputs(reads(input)) }
+      each(stdin_arguments + arguments) { |input| outputs(reads(input)) }
+    end
+
+    def everything
+      read_stdin_files + stdin_arguments + read_files + arguments
     end
 
     def command_each
-      each(read_stdin_files + read_files + stdin_arguments) do |input|
-        each(lines(reads(input))) do |line|
-          evaluates(arguments.join(" "), it: line)
-        end
+      each(everything) do |input|
+        each(lines(reads(input)), progress: true) { |line| evaluates(it: line) }
+      end
+    end
+
+    def command_tally
+      each(everything) do |input|
+        outputs(
+          JSON.pretty_generate(
+            map(lines(reads(input)), progress: true) do |element|
+              if ruby.to_s.empty?
+                element
+              else
+                evaluates(it: element, returns: true, stdout: false).returned
+              end
+            end.tally
+          )
+        )
+      end
+    end
+
+    def command_all
+      each(everything, progress: true) { |input| evaluates(it: reads(input)) }
+    end
+
+    def command_append
+      outputs(everything.map { |input| lines(reads(input)) }.inject(&:+))
+    end
+
+    def command_prepend
+      outputs(
+        everything.reverse.map { |input| lines(reads(input)) }.inject(&:+)
+      )
+    end
+
+    def command_select
+      each(stdin_files + files) do |input|
+        outputs(
+          select(lines(reads(File.read(input)))) { |element| match?(element) },
+          file: input
+        )
+      end
+
+      each(stdin_arguments + arguments) do |input|
+        outputs(select(lines(reads(input))) { |element| match?(element) })
+      end
+    end
+
+    def command_reject
+      each(stdin_files + files) do |input|
+        outputs(
+          reject(lines(reads(File.read(input)))) { |element| match?(element) },
+          file: input
+        )
+      end
+
+      each(stdin_arguments + arguments) do |input|
+        outputs(reject(lines(reads(input))) { |element| match?(element) })
+      end
+    end
+
+    def command_after
+      each(stdin_files + files) do |input|
+        outputs(after(lines(reads(File.read(input)))), file: input)
+      end
+
+      each(stdin_arguments + arguments) do |input|
+        outputs(after(lines(reads(input))))
+      end
+    end
+
+    def command_before
+      each(stdin_files + files) do |input|
+        outputs(before(reads(File.read(input))), file: input)
+      end
+
+      each(stdin_arguments + arguments) do |input|
+        outputs(before(lines(reads(input))))
+      end
+    end
+
+    def command_between
+      each(stdin_files + files) do |input|
+        outputs(between(lines(reads(File.read(input)))), file: input)
+      end
+
+      each(stdin_arguments + arguments) do |input|
+        outputs(between(lines(reads(input))))
+      end
+    end
+
+    def command_anonymize
+      each(stdin_files + files) do |input|
+        outputs(anonymize(reads(File.read(input))), file: input)
+      end
+
+      each(stdin_arguments + arguments) do |input|
+        outputs(anonymize(reads(input)))
       end
     end
 
@@ -141,25 +352,22 @@ class Dorian
 
       case output
       when :csv
-        CSV.generate(headers: headers_of(content)) do |csv|
-          csv << headers_of(content) if headers_of(content)
-
-          each(content) { |row| csv << row }
-        end
+        (headers_of(content) ? headers_of(content).to_csv : "") +
+          map(content) do |element|
+            CSV.generate(headers: headers_of(content)) do |csv|
+              csv << wrap(element)
+            end
+          end.join
       when :json
         pretty? ? JSON.pretty_generate(content) : content.to_json
-      when :jsonl
+      when :jsonl, :yamll
         map(content, &:to_json).join("\n")
       when :raw
         content
-      when :ruby
-        content.inspect
-      when :rubyl
-        map(content, &:inspect).join("\n")
       when :yaml
         content.to_yaml
       else
-        abort "#{output} not supported"
+        abort "#{output.inspect} not supported"
       end
     end
 
@@ -173,15 +381,19 @@ class Dorian
         end
       when :json
         JSON.parse(content).to_deep_struct
-      when :jsonl
+      when :jsonl, :yamll
         map(content.lines) { |line| JSON.parse(line) }.to_deep_struct
       when :raw
         content
       when :yaml
-        YAML.safe_load(content)
+        YAML.safe_load(content).to_deep_struct
       else
-        abort "#{input} not supported"
+        abort "#{input.inspect} not supported"
       end
+    rescue JSON::ParserError => e
+      abort "invalid json: #{e.message}"
+    rescue Psych::SyntaxError => e
+      abort "invalid yaml: #{e.message}"
     end
 
     def pretty?
@@ -190,13 +402,15 @@ class Dorian
 
     def read_stdin
       @read_stdin ||= $stdin.each_line.to_a
+    rescue Interrupt
+      abort "interupt in read_stdin"
     end
 
     def stdin_files
-      return [] if files.any?
-      return [] unless stdin == :files
+      return [] if files.any? || arguments.any?
+      return [] if stdin != :files
 
-      read_stdin.map(&:rstrip)
+      map(read_stdin, &:rstrip)
     end
 
     def read_stdin_files
@@ -208,7 +422,7 @@ class Dorian
     end
 
     def stdin_arguments
-      return [] if files.any?
+      return [] if files.any? || arguments.any?
       return [] if stdin == :files
 
       [read_stdin.join]
@@ -218,6 +432,10 @@ class Dorian
       options.stdin.to_sym
     end
 
+    def deep?
+      !!options.deep
+    end
+
     def options
       parsed.options
     end
@@ -294,30 +512,252 @@ class Dorian
       parsed.help
     end
 
+    def n
+      options.n
+    end
+
+    def parallel_type
+      options.parallel_type&.to_sym
+    end
+
     def headers_of(content)
-      return unless headers?
       return unless content.respond_to?(:first)
       return unless content.first
       return unless content.first.respond_to?(:to_h)
       return unless content.first.to_h.keys.any?
 
       content.first.to_h.keys
+    rescue TypeError
+      nil
     end
 
-    def each(collection, &)
-      parallel? ? Parallel.each(collection, &) : collection.each(&)
+    def parallel_options
+      if parallel_type == :processes
+        { in_processes: n }
+      elsif parallel_type == :threads
+        { in_threads: n }
+      else
+        abort "#{parallel_type.inspect} not supported"
+      end
     end
 
-    def map(collection, &)
-      parallel? ? Parallel.map(collection, &) : collection.map(&)
+    def each(collection, options: parallel_options, progress: false, &)
+      collection = wrap(collection)
+      progress_bar = progress ? create_progress_bar(collection.size) : nil
+
+      if parallel?
+        Parallel.each(
+          collection,
+          **options,
+          finish: ->(*) { progress_bar&.increment },
+          &
+        )
+      else
+        collection.each do |element|
+          yield(element).tap { progress_bar&.increment }
+        end
+      end
+    end
+
+    def map(collection, options: parallel_options, progress: false, &)
+      collection = wrap(collection)
+      progress_bar = progress ? create_progress_bar(collection.size) : nil
+
+      if parallel?
+        Parallel.map(
+          collection,
+          **options,
+          finish: ->(*) { progress_bar&.increment },
+          &
+        )
+      else
+        collection.map do |element|
+          yield(element).tap { progress_bar&.increment }
+        end
+      end
+    end
+
+    def select(collection, progress: false, &)
+      collection = wrap(collection)
+      progress_bar = progress ? create_progress_bar(collection.size) : nil
+
+      collection.select do |element|
+        yield(element).tap { progress_bar&.increment }
+      end
+    end
+
+    def reject(collection, progress: false, &)
+      collection = wrap(collection)
+      progress_bar = progress ? create_progress_bar(collection.size) : nil
+
+      collection.reject do |element|
+        yield(element).tap { progress_bar&.increment }
+      end
     end
 
     def lines(input)
-      input.is_a?(String) ? input.lines.map(&:rstrip) : Array(input)
+      if input.is_a?(String)
+        input.lines.map(&:rstrip)
+      elsif deep?
+        deep_lines(input)
+      else
+        Array(input)
+      end
+    end
+
+    def deep_lines(input)
+      case input
+      when Array
+        [input.to_deep_struct] +
+          input.flat_map { |element| deep_lines(element) }
+      when Hash
+        [input.to_deep_struct] +
+          input.flat_map { |key, value| deep_lines([key, value]) }
+      when Struct
+        deep_lines(input.from_deep_struct).to_deep_struct
+      else
+        [input.to_deep_struct]
+      end
+    end
+
+    def wrap(ruby)
+      if ruby.is_a?(Hash)
+        ruby
+      elsif ruby.respond_to?(:to_a)
+        ruby.to_a
+      else
+        Array(ruby)
+      end
+    end
+
+    def create_progress_bar(total)
+      return unless progress?
+
+      Dorian::Progress.create(total:, format: progress_format)
+    end
+
+    def after(input, ruby: @ruby_after || @ruby)
+      if ruby.to_i.to_s == ruby
+        input[(ruby.to_i)..]
+      else
+        selected = false
+
+        input.select do |element|
+          selected = true if match?(element, ruby:)
+          selected
+        end
+      end
+    end
+
+    def before(input, ruby: @ruby_before || @ruby)
+      if ruby.to_i.to_s == ruby
+        input[..(ruby.to_i)]
+      else
+        selected = true
+
+        input.select do |element|
+          selected.tap { selected = false if match?(element, ruby:) }
+        end
+      end
+    end
+
+    def between(input, ruby_before: @ruby_before, ruby_after: @ruby_after)
+      if ruby_before.to_i.to_s == ruby_before &&
+           ruby_after.to_i.to_s == ruby_after
+        input[(ruby_after.to_i)..(ruby_before.to_i)]
+      else
+        selected = false
+
+        input.select do |element|
+          selected = true if match?(element, ruby: ruby_after)
+          selected.tap do
+            selected = false if match?(element, ruby: ruby_before)
+          end
+        end
+      end
+    end
+
+    def anonymize(input)
+      if input.is_a?(String)
+        input.gsub(/[a-z]/, "a").gsub(/[A-Z]/, "A").gsub(/[0-9]/, "0")
+      elsif input.is_a?(Integer)
+        0
+      elsif input.is_a?(Float)
+        0.0
+      elsif input.is_a?(TrueClass) || input.is_a?(FalseClass)
+        false
+      elsif input.nil?
+        nil
+      elsif input.is_a?(Hash)
+        input.transform_values { |value| anonymize(value) }
+      elsif input.is_a?(Array)
+        input.map { |element| anonymize(element) }
+      elsif input.is_a?(Struct)
+        anonymize(input.from_deep_struct).to_deep_struct
+      else
+        raise "#{input.class.inspect} not supported"
+      end
+    end
+
+    def match?(element, ruby: @ruby)
+      !!evaluates(ruby:, it: element, stdout: false, returns: true).returned
+    end
+
+    def token(file)
+      token_file = File.join(Dir.home, file)
+
+      if File.exist?(token_file)
+        token = File.read(token_file).strip
+      else
+        print "token: "
+        token = gets.strip
+        File.write(token_file, token)
+        puts "token written to #{token_file}"
+      end
+
+      token
+    end
+
+    def completion(token:, model:, messages:)
+      body =
+        post(
+          "https://api.openai.com/v1/chat/completions",
+          headers: {
+            "Content-Type" => "application/json",
+            "Authorization" => "Bearer #{token}"
+          },
+          body: { model:, messages: }.to_json
+        )
+
+      json = JSON.parse(body)
+      output = json.dig("choices", 0, "message", "content")
+
+      if output
+        output.strip
+      else
+        abort JSON.pretty_generate(json)
+      end
+    end
+
+    def post(url, headers: {}, body: {})
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      request = Net::HTTP::Post.new(uri.path, headers)
+      request.body = body
+      http.request(request).body
+    end
+
+    def short(string)
+      string[0..5000]
+    end
+
+    def encoder
+      Tiktoken.encoding_for_model("gpt-4o")
     end
 
     def evaluates(
-      ruby,
+      ruby: @ruby,
       it: nil,
       debug: debug?,
       stdout: stdout?,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dorian
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.2.0
 platform: ruby
 authors:
 - Dorian Marié
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-31 00:00:00.000000000 Z
+date: 2024-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: git
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement