aliyun-log 0.2.12 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba1bcbd83277ca8da27978ecd7a603a205a19c1600b11f359135e2629e6f61c4
-  data.tar.gz: d1e731f6b181167ef11004c7c79bbbef9acd290be531a08a24bb6f87134cfaac
+  metadata.gz: 5d6574ee8e72a65c178f4cdc28c61e63379999e51166c95756b9ffd6b367f47a
+  data.tar.gz: 8fdce159cd971c94c966b20db9926653a16e4bd5efaee918f619a6f0bf00a9e8
 SHA512:
-  metadata.gz: 850989116e4820287e15fbe5361319f1043d4a1c3e7991bab947187bcac2d0114ee9244df26c4c132cd29d400752b4f18a6c0a3bcccaaabdda151858829b19e4
-  data.tar.gz: c03afaddd81499c4cb36be357d9894eccf2574fa009afa2d57f7d01e61885534a60b94f4bc2dfaa315e5db0862481225966fc71f27fb8ce1d6a1686284f90e40
+  metadata.gz: 13ce9aed0a16e0c0a47fe8baadc45215967fd33cef33bece8bdfca0148a565155f347860d3bf986091b965419e04d513e19756c87b6f52ac32879188f07f3a11
+  data.tar.gz: f7c441bdb8ae2f3a768b236fb1bc8b86ad0c4965324b9323142565d50ed776b632ff11ba2036af7b1d1058968daeabe14080f792cbf2d438c777a6f4a25f17cf

data/lib/aliyun/log/protobuf.rb

@@ -16,11 +16,17 @@ module Aliyun
         repeated Content, :contents, 2
       end
 
+      class LogTag < ::Protobuf::Message
+        required :string, :key, 1
+        required :string, :value, 2
+      end
+
       class LogGroup < ::Protobuf::Message
         repeated Log, :logs, 1
         optional :string, :reserved, 2
         optional :string, :topic, 3
         optional :string, :source, 4
+        repeated LogTag, :log_tags, 6
       end
     end
   end

data/lib/aliyun/log/record.rb

@@ -4,7 +4,7 @@ require 'active_support'
 require 'active_support/time'
 require 'active_support/core_ext'
 require 'active_model'
-require 'forwardable'
+require 'monitor'
 
 require_relative 'record/exception'
 require_relative 'record/field'
@@ -22,16 +22,21 @@ module Aliyun
 
         class_attribute :options, instance_accessor: false, default: {}
         class_attribute :base_class, instance_accessor: false, default: self
-        class_attribute :log_connection, instance_accessor: false
         class_attribute :_schema_load, default: false
 
         Log.included_models << self unless Log.included_models.include? self
 
+        field :__time__, type: :long, cast_type: :integer
+        field :__topic__
+        field :__source__
+
         field :created_at, type: :text, cast_type: :datetime if Config.timestamps
 
         define_model_callbacks :save, :create, :initialize
 
         before_save :set_created_at
+
+        @lock = Monitor.new
       end
 
       include Field
@@ -79,6 +84,10 @@ module Aliyun
         end
       end
 
+      def new_record?
+        @new_record == true
+      end
+
       def inspect
         inspection = if defined?(@attributes) && @attributes
                        self.class.attributes.keys.collect do |name|

data/lib/aliyun/log/record/field.rb

@@ -20,6 +20,7 @@ module Aliyun
 
         included do
           class_attribute :attributes, instance_accessor: false, default: {}
+          class_attribute :tag_attributes, instance_accessor: false, default: {}
           extend Common::Logging
         end
 
@@ -31,7 +32,11 @@ module Aliyun
             end
 
             named = name.to_s
-            self.attributes = attributes.merge(name => { type: type }.merge(options))
+            if options[:log_tag]
+              self.tag_attributes = tag_attributes.merge(name => { type: type }.merge(options))
+            else
+              self.attributes = attributes.merge(name => { type: type }.merge(options))
+            end
 
             warn_about_method_overriding(name, name)
             warn_about_method_overriding("#{named}=", name)

data/lib/aliyun/log/record/persistence.rb

@@ -4,6 +4,14 @@ module Aliyun
   module Log
     module Record
       module Persistence
+        RESERVED_FIELDS = %i[
+          __time__
+          __topic__
+          __source__
+          __partition_time__
+          __extract_others__
+        ].freeze
+
         extend ActiveSupport::Concern
 
         module ClassMethods
@@ -40,11 +48,12 @@ module Aliyun
           end
 
           def auto_load_schema
-            return if _schema_load
-
-            create_logstore
-            sync_index
-            self._schema_load = true
+            @lock.synchronize do
+              return if _schema_load
+              create_logstore
+              sync_index
+              self._schema_load = true
+            end
           end
 
           def has_index?
@@ -54,19 +63,23 @@ module Aliyun
             false
           end
 
-          def create(data, opts = {})
+          def create(data, force = false)
             auto_load_schema
             if data.is_a?(Array)
-              logs = data.map do |log_attr|
-                new(log_attr).save_array
+              # TODO batch insert
+              data.each do |log|
+                saved = new(log).save(force)
+                return false unless saved
               end
-              res = Log.record_connection.put_log(project_name, logstore_name, logs, opts)
-              res.code == 200
             else
-              new(data).save
+              new(data).save(force)
             end
           end
 
+          def create!(data)
+            create(data, true)
+          end
+
           private
 
           def evaluate_default_value(val)
@@ -80,11 +93,12 @@ module Aliyun
           end
 
           def field_indices
-            if options[:field_index] == false
-              attributes.select { |_, value| value[:index] == true }
-            else
-              attributes.reject { |_, value| value[:index] == false }
-            end
+            indices = if options[:field_index] == false
+                        attributes.select { |_, value| value[:index] == true }
+                      else
+                        attributes.reject { |_, value| value[:index] == false }
+                      end
+            indices.reject { |key, _| RESERVED_FIELDS.include?(key) }
           end
 
           def create_index
@@ -125,38 +139,68 @@ module Aliyun
           end
         end
 
+        def dump_log_tags
+          log_tags = {}
+          self.class.tag_attributes.map do |key, options|
+            log_tags[key] = TypeCasting.dump_field(attributes[key], options)
+          end
+          log_tags.compact
+        end
+
         def dump_attributes
           attributes.dup.tap do |tap|
             tap.each do |k, v|
-              tap[k] = TypeCasting.dump_field(v, self.class.attributes[k])
+              if self.class.attributes[k][:log_tag] || RESERVED_FIELDS.include?(k)
+                tap.delete(k)
+              else
+                tap[k] = TypeCasting.dump_field(v, self.class.attributes[k])
+              end
             end
           end
         end
 
-        def save
+        def save(force = false)
           self.class.auto_load_schema
           run_callbacks(:create) do
             run_callbacks(:save) do
-              if valid?
-                res = Log.record_connection.put_log(
-                  self.class.project_name,
-                  self.class.logstore_name,
-                  dump_attributes
-                )
-                res.code == 200
-              else
-                false
-              end
+              force && validate!
+              return false unless valid?
+              saved = put_logs
+              @new_record = false if saved
+              saved
             end
           end
         end
 
-        def save_array
-          run_callbacks(:create) do
-            run_callbacks(:save) do
-              validate! && dump_attributes
-            end
-          end
+        def save!
+          save(true)
+        end
+
+        private
+
+        def put_logs
+          log_tags = []
+          dump_log_tags.each do |k, v|
+            log_tags << Protobuf::LogTag.new(key: k, value: v)
+          end
+          lg = Protobuf::LogGroup.new(
+            logs: [generate_log],
+            log_tags: log_tags,
+            topic: read_attribute(:__topic__),
+            source: read_attribute(:__source__)
+          )
+          res = Log.record_connection.put_logs(
+            self.class.project_name,
+            self.class.logstore_name,
+            lg
+          )
+          res.code == 200
+        end
+
+        def generate_log
+          time = read_attribute(:__time__) || Time.now.to_i
+          contents = dump_attributes.map { |k, v| { key: k, value: v.to_s } }
+          Protobuf::Log.new(time: time, contents: contents)
         end
       end
     end

data/lib/aliyun/log/record/relation.rb

@@ -216,7 +216,7 @@ module Aliyun
           sql_query << "ORDER BY #{opts[:order]}" if opts[:order].present?
           if sql_query.present? || opts[:select].present?
             sql_query.insert(0, "SELECT #{opts[:select] || '*'}")
-            sql_query.insert(1, 'FROM log')
+            sql_query.insert(1, 'FROM log') unless opts[:select]&.match?(/from\s+log/i)
             if opts[:line] || opts[:page] || opts[:offset]
               parse_page
               sql_query << "LIMIT #{@opts[:offset]},#{@opts[:line]}"
@@ -250,7 +250,7 @@ module Aliyun
         def sanitize_hash(search_content)
           return search_content unless search_content.is_a?(Hash)
 
-          search_content.select { |_, v| v.present? }.map do |key, value|
+          search_content.reject { |_, v| v.nil? }.map do |key, value|
             options = @klass.attributes[:"#{key}"]
             unless options
               raise UnknownAttributeError, "unknown field '#{key}' for #{@klass.name}."
@@ -259,14 +259,18 @@ module Aliyun
             raise_if_hash_quote(value)
 
             cast_type = options[:cast_type]
+
+            is_tag = @klass.tag_attributes[:"#{key}"]
+            key = :"__tag__:#{key}" if is_tag
             if value.is_a?(Array)
-              values = value.uniq.map { |v| _quote(cast_type, v) }
+              values = value.uniq.map { |v| is_tag ? v : _quote(cast_type, v) }
               str_values = values.map { |v| "#{key}: #{v}" }.join(' OR ')
               values.size > 1 ? "(#{str_values})" : str_values
             elsif value.is_a?(Range)
               "#{key} in [#{value.begin} #{value.end}]"
             else
-              "#{key}: #{_quote(cast_type, value)}"
+              quote_value = is_tag ? value : _quote(cast_type, value)
+              "#{key}: #{quote_value}"
             end
           end.join(' AND ')
         end

data/lib/aliyun/log/record/type_casting.rb

@@ -55,7 +55,17 @@ module Aliyun
           end
         end
 
-        class StringType < Value; end
+        class StringType < Value
+          def cast(value)
+            if value.nil?
+              ''
+            elsif value.is_a? String
+              value.dup
+            else
+              value.to_s
+            end
+          end
+        end
 
         class DateType < Value
           def cast(value)

data/lib/aliyun/version.rb

@@ -2,6 +2,6 @@
 
 module Aliyun
   module Log
-    VERSION = '0.2.12'
+    VERSION = '0.2.13'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aliyun-log
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.2.13
 platform: ruby
 authors:
 - Yingce Liu
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-13 00:00:00.000000000 Z
+date: 2020-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel