llmed 0.1.17 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66603fc5fdc0f63f5c066ca085a80dee2a259e4a1d30f315f52e018f5bfec0de
-  data.tar.gz: 244a8d4938c85cd395a541eb18491e007b8e116d4dbf997d4f71a05050d5b5a9
+  metadata.gz: cc7a4dd77be914e3acf61278c46190acfc65ceec923b6c1041a29b6fd488b623
+  data.tar.gz: 6ef9e5985f6635126689d0793be72511c72f598579bfd73955c9d1fcbfc2fd14
 SHA512:
-  metadata.gz: 2f9c7c6e0f9949b7a2876051912a645e786bd65eaba09e23bf837f1e52ae1cb8c0d645194028ac5f8b65c5e0a0f8d5d6197d6ac1155753dfa38f887ac57a6fc9
-  data.tar.gz: 308e33e710860f4ed3b63b007875aa4c339edac3bbb9f8ab6c13452cb48919408e610282991f53fefb38c27a009fb3e9acf0e4035944f40867a9f833825b2e49
+  metadata.gz: dfab97f3a34b02fad0bc1e7416031c2e2dbc8c63b4d53e225eae40addbeed057ec90e7e2dfcc9ed02a923382ea29fe96e640c16af35baa73b6ffcb3dbe6e2caf
+  data.tar.gz: e64afcb33ce327c3099a103afb09f65591c2b343f4d98a6b818a6e93d371053e44162933f8f709e47effae01cd856c865b4060391d20b9d8ed1696ed137ee509

data/lib/llmed.rb

@@ -3,6 +3,7 @@
 
 require 'pp'
 require 'csv'
+require 'digest'
 require 'json'
 require 'pathname'
 require 'fileutils'
@@ -24,8 +25,16 @@ class LLMed
       @skip
     end
 
+    def same_digest?(val)
+      digest == val
+    end
+
+    def digest
+      Digest::SHA256.hexdigest "#{@name}.#{@message}"
+    end
+
     def message
-      "# #{@name}\n\n#{@message}"
+      "# Context: #{@name} Digest: #{digest}\n\n#{@message}"
     end
 
     def llm(message)
@@ -67,11 +76,19 @@ Always include the properly escaped comment: LLMED-COMPILED.
 You must only modify the following source code:
 {source_code}
 
-", input_variables: %w[language source_code])
+Only generate source code of the context who digest belongs to {update_context_digests}.
+
+Wrap with comment every code that belongs to the indicated context, example in ruby:
+#<llmed-code context='context name' digest='....'>
+...
+#</llmed-code>
+
+", input_variables: %w[language source_code update_context_digests])
     end
 
-    def prompt(language:, source_code:)
-      @prompt.format(language: language, source_code: source_code)
+    def prompt(language:, source_code:, update_context_digests: [])
+      @prompt.format(language: language, source_code: source_code,
+                     update_context_digests: update_context_digests.join(','))
     end
 
     # Change the default prompt, input variables: language, source_code
@@ -164,6 +181,15 @@ You must only modify the following source code:
       File.read(release_source_code)
     end
 
+    def release_contexts(_output_dir, release_dir)
+      return {} unless @release
+
+      release_source_code = Pathname.new(release_dir) + "#{@output_file}.r#{@release}#{@language}.cache"
+      return {} unless File.exist?(release_source_code)
+
+      File.read(release_source_code).scan(/context='(.+)' digest='(.+)'/).to_h
+    end
+
     def output_file(output_dir, mode = 'w', &block)
       if @output_file.respond_to? :write
         yield @output_file
@@ -177,6 +203,68 @@ You must only modify the following source code:
       end
     end
 
+    def patch_or_create(output_dir, release_dir, output)
+      release_source_code_path = Pathname.new(release_dir) + "#{@output_file}.r#{@release}#{@language}.cache"
+
+      if @release && File.exist?(release_source_code_path)
+        release_source_code = File.read(release_source_code_path)
+        output_contexts = output.scan(%r{<llmed-code context='(.+?)' digest='(.+?)'>(.+?)</llmed-code>}im)
+        output_contexts.each do |match|
+          name, digest, new_code = match
+          @logger.info("APPLICATION #{@name} PATCHING CONTEXT #{name}")
+          release_source_code = release_source_code.sub(%r{(.*?)(<llmed-code context='#{name}' digest='.*?'>)(.+?)(</llmed-code>)(.*?)}m) do
+            "#{::Regexp.last_match(1)}<llmed-code context='#{name}' digest='#{digest}'>#{new_code}#{::Regexp.last_match(4)}#{::Regexp.last_match(5)}"
+          end
+        end
+
+        output_file(output_dir) do |file|
+          file.write(release_source_code)
+        end
+      else
+        output_file(output_dir) do |file|
+          file.write(output)
+        end
+      end
+    end
+
+    def digests_of_context_to_update(output_dir, release_dir)
+      update_context_digest = []
+      release_contexts = release_contexts(output_dir, release_dir)
+      unless release_contexts.empty?
+        # rebuild context from top to down
+        # we are expecting:
+        # - top the most stable concepts
+        # - buttom the most inestable concepts
+        update_rest = false
+        @contexts.each do |ctx|
+          release_context_digest = release_contexts[ctx.name]
+          if update_rest
+            update_context_digest << release_context_digest
+            next
+          end
+          next if ctx.same_digest?(release_context_digest)
+
+          update_rest = true
+          update_context_digest << release_context_digest
+        end
+      end
+
+      update_context_digest
+    end
+
+    def rebuild?(output_dir, release_dir)
+      return true unless @release
+
+      update_context_digest = digests_of_context_to_update(output_dir, release_dir)
+      release_contexts = release_contexts(output_dir, release_dir)
+      update_context_digest.each do |digest|
+        context_by_digest = release_contexts.invert
+        @logger.info("APPLICATION #{@name} REBUILDING CONTEXT #{context_by_digest[digest]}")
+      end
+
+      !update_context_digest.empty?
+    end
+
     def write_statistics(release_dir, response)
       return unless @output_file.is_a?(String)
 
@@ -240,33 +328,38 @@ You must only modify the following source code:
     @logger.info("APPLICATION #{app.name} COMPILING")
 
     llm = @configuration.llm
+
+    app.evaluate
+
     system_content = @configuration.prompt(language: app.language,
                                            source_code: app.source_code(
                                              output_dir, release_dir
-                                           ))
+                                           ),
+                                           update_context_digests: app.digests_of_context_to_update(output_dir,
+                                                                                                    release_dir))
     messages = [LLMed::LLM::Message::System.new(system_content)]
-    app.evaluate
     app.contexts.each do |ctx|
       next if ctx.skip?
 
       messages << LLMed::LLM::Message::User.new(ctx.message)
     end
-
-    llm_response = llm.chat(messages: messages)
-    @logger.info("APPLICATION #{app.name} TOTAL TOKENS #{llm_response.total_tokens}")
-    write_output(app, output_dir, llm_response.source_code)
-    write_statistics(app, release_dir, llm_response)
-    app.notify("COMPILE DONE #{llm_response.duration_seconds}")
+    if app.rebuild?(output_dir, release_dir)
+      llm_response = llm.chat(messages: messages)
+      @logger.info("APPLICATION #{app.name} TOTAL TOKENS #{llm_response.total_tokens}")
+      write_output(app, output_dir, release_dir, llm_response.source_code)
+      write_statistics(app, release_dir, llm_response)
+      app.notify("COMPILE DONE #{llm_response.duration_seconds}")
+    else
+      @logger.info("APPLICATION #{app.name} NOT CHANGES DETECTED")
+    end
   end
 
   def write_statistics(app, release_dir, response)
     app.write_statistics(release_dir, response)
   end
 
-  def write_output(app, output_dir, output)
-    app.output_file(output_dir) do |file|
-      file.write(output)
-    end
+  def write_output(app, output_dir, release_dir, output)
+    app.patch_or_create(output_dir, release_dir, output)
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llmed
 version: !ruby/object:Gem::Version
-  version: 0.1.17
+  version: 0.2.0
 platform: ruby
 authors:
 - Jovany Leandro G.C