daifuku 0.9.0

data/example/iOS/generate-log-classes.rb

@@ -0,0 +1,212 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __dir__)
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+require 'daifuku'
+require 'erb'
+require 'active_support/inflector'
+require 'fileutils'
+
+ROOT_PATH = File.absolute_path(File.join(__dir__, '../'))
+LOG_DEFINITIONS = File.join(ROOT_PATH, 'LogDefinitions')
+TEMPLATE_DIR = File.join(ROOT_PATH, 'iOS/Templates')
+GENERATED_CODE_DIR = File.join(ROOT_PATH, 'AutoGenerated')
+SWIFT_TYPE_MAPS = {
+  'smallint' => 'Int16',
+  'integer' => 'Int',
+  'bigint' => 'Int64',
+  'real' => 'Float',
+  'double' => 'Double',
+  'boolean' => 'Bool',
+  'string' => 'String',
+  'date' => 'JSTDateColumn',
+  'timestampz' => 'ZonedTimestampColumn'
+}
+UNSUPPORTED_TYPES = %w(timestamp)
+
+# Represents a Category consisting of multiple events
+# ex) Category: RecipeDetail
+class CategoryRepresentation
+  def initialize(category)
+    @category = category
+  end
+
+  def name
+    @category.name
+  end
+
+  def class_name
+    @category.name.camelize
+  end
+
+  def descriptions
+    @category.descriptions
+  end
+
+  def variable_name
+    @category.name.camelize(:lower)
+  end
+
+  def events
+    @events ||= @category.events.map { |_, event| EventRepresentation.new(event) }
+  end
+
+  def available_events
+    events.reject(&:obsolete?)
+  end
+end
+
+# Represents a Event consisting of multiple columns
+# ex) Event: tap_sample_view
+class EventRepresentation
+  def initialize(event)
+    @event = event
+  end
+
+  def name
+    @event.name
+  end
+
+  def variable_name
+    @event.name.camelize(:lower)
+  end
+
+  def columns
+    @columns ||= @event.columns.reject(&:obsolete?).map { |column| ColumnRepresentation.new(column) }
+  end
+
+  def descriptions
+    @event.descriptions
+  end
+
+  def associated_types
+    columns.map(&:as_argument)&.join(', ')
+  end
+
+  def pattern_matches
+    columns.map { |column| column.variable_name.to_string }&.join(', ')
+  end
+
+  def availability_annotation
+    obsolete? ? '@available(*, unavailable) ' : ''
+  end
+
+  def obsolete?
+    @event.obsolete?
+  end
+end
+
+# Represents a column
+class ColumnRepresentation
+  def initialize(column)
+    @column = column
+  end
+
+  def variable_name
+    @column.name.camelize(:lower)
+  end
+
+  def original_name
+    @column.name
+  end
+
+  def swift_type
+    convert_to_swift_type(@column)
+  end
+
+  def descriptions
+    @column.descriptions
+  end
+
+  def as_argument
+    "#{variable_name}: #{swift_type}"
+  end
+
+  def call_dump
+    caller = if @column.type.optional?
+               "#{variable_name}?"
+             else
+               variable_name.to_s
+             end
+    type = @column.type
+    if type.name == 'string' && !type.str_length.nil?
+      "#{caller}.validateLength(within: #{type.str_length}).dump()"
+    else
+      "#{caller}.dump()"
+    end
+  end
+
+  private
+
+  def convert_to_swift_type(column)
+    type = column.type
+    raise "Code builder currently doesn't support #{type.name}." if UNSUPPORTED_TYPES.include?(type.name)
+
+    if type.custom?
+      swift_type ||= type.name # custom type
+    else
+      swift_type = SWIFT_TYPE_MAPS[type.name] # primitive types
+    end
+    if type.optional?
+      "#{swift_type}?"
+    else
+      swift_type
+    end
+  end
+end
+
+# Generating Swift files from Markdown files using Daifuku
+class Generator
+  def initialize
+    check_category_names
+    @common_category = categories[COMMON_CATEGORY_NAME]
+    raise 'Could not found common category. Please define common.md' unless @common_category
+  end
+
+  def generate_common_payload!(destination)
+    FileUtils.mkdir_p(File.dirname(destination))
+    template = IO.read(File.join(TEMPLATE_DIR, 'CommonPayload.swift.erb'))
+    columns = @common_category.common_columns.reject(&:obsolete?).map { |column| ColumnRepresentation.new(column) }
+    result = render(template, columns: columns)
+    IO.write(destination, result)
+  end
+
+  def generate_all_log_categories!(destination)
+    FileUtils.mkdir_p(File.dirname(destination))
+    template = IO.read(File.join(TEMPLATE_DIR, 'LogCategories.swift.erb'))
+    categories_to_generate = categories.map { |_, category| CategoryRepresentation.new(category) }
+    result = render(template, categories: categories_to_generate)
+    IO.write(destination, result)
+  end
+
+  def check_category_names
+    # Daifuku uses the file name as the category name, the category name in the Markdown file is simply ignored.
+    # But for consistency, here we make sure they are the same.
+    Dir.glob(File.join(LOG_DEFINITIONS, '*.md')) do |path|
+      content = File.read(path)
+      md_category_name = /^#\s*(.+)/.match(content)
+      raise "Could not find category name in #{path}" unless md_category_name
+
+      category_name = md_category_name[1]
+      file_name = File.basename(path, '.md')
+      unless file_name == category_name
+        raise %(The file name "#{file_name}" does not match the category name "#{category_name}" in #{path})
+      end
+    end
+  end
+
+  private
+
+  def render(template, hash)
+    ERB.new(template, trim_mode: '-').result_with_hash(hash)
+  end
+
+  def categories
+    @categories ||= Daifuku::Compiler.new.compile(LOG_DEFINITIONS)
+  end
+end
+
+generator = Generator.new
+generator.generate_common_payload!(File.join(GENERATED_CODE_DIR, 'CommonPayload.swift'))
+generator.generate_all_log_categories!(File.join(GENERATED_CODE_DIR, 'LogCategories.swift'))

data/lib/daifuku/compiler.rb

@@ -0,0 +1,43 @@
+module Daifuku
+  DEFAULT_NAME_LENGTH = 64
+
+  class Compiler
+    attr_reader :options
+
+    # Compile markdowns into Ruby objects.
+    # @param directory_path String Directory path containing log definitions.
+    # @param options [String: Object] Compiler options
+    # @return [String: Category]
+    def compile(directory_path, options = {})
+      @options = options
+      categories = Dir.entries(directory_path).map do |file|
+        if file.end_with?('.md')
+          category_name = File.basename(file, '.md')
+          path = File.join(directory_path, file)
+          # This situation implicates missing LANG, use UTF-8 by default.
+          enc = (Encoding.default_external == Encoding::US_ASCII) ? Encoding::UTF_8 : Encoding.default_external
+          str = File.read(path, encoding: enc)
+          parser.parse(str, category_name)
+        end
+      end.compact.map { |category| [category.name, category] }.to_h
+      validator.validate!(categories)
+      categories
+    end
+
+    def parser
+      @parser ||= Parser.new
+    end
+
+    private
+    def validator
+      @validator ||= Validator.new([MultipleTypesRule.new,
+                                    ReservedColumnsRule.new,
+                                    ShadowingRule.new,
+                                    NameLengthRule.new(name_length)])
+    end
+
+    def name_length
+      options[:name_length] || DEFAULT_NAME_LENGTH
+    end
+  end
+end

data/lib/daifuku/models.rb

@@ -0,0 +1,171 @@
+module Daifuku
+  class Category
+    # @return String
+    attr_reader :name
+    # @return [String: Event]
+    attr_reader :events
+    # @return String
+    attr_reader :descriptions
+    # @return [Column]
+    attr_reader :common_columns
+
+    def initialize(name, events, descriptions, common_columns)
+      @name = name
+      @events = events
+      @descriptions = descriptions
+      @common_columns = common_columns
+    end
+
+    def dump
+      {
+        'name' => name,
+        'events' => events.to_h { |name, event| [name, event.dump] },
+        'descriptions' => descriptions,
+        'common_columns' => common_columns.map(&:dump),
+      }
+    end
+  end
+
+  class Event
+    # @return String
+    attr_reader :name
+    # @return [Column]
+    attr_reader :columns
+    # @return String
+    attr_reader :descriptions
+
+    def obsolete?
+      @is_obsolete
+    end
+
+    def initialize(name, columns, descriptions, is_obsolete)
+      @name = name
+      @columns = columns
+      @descriptions = descriptions
+      @is_obsolete = is_obsolete
+    end
+
+    def dump
+      {
+        'name' => name,
+        'columns' => columns.map(&:dump),
+        'descriptions' => descriptions,
+        'is_obsolete' => obsolete?,
+      }
+    end
+  end
+
+  class Column
+    # @return String
+    attr_reader :name
+    # @return Type
+    attr_reader :type
+    # @return [String]
+    attr_reader :descriptions
+
+    def obsolete?
+      @is_obsolete
+    end
+
+    def initialize(name, type, descriptions, is_obsolete = false)
+      @name = name
+      @type = type
+      @descriptions = descriptions
+      @is_obsolete = is_obsolete
+    end
+
+    def self.parse(descriptor, descriptions)
+      # column_name: !bigint
+      # column_name: !bigint?
+      # column_name: !string 256
+      # column_name: !string? 256
+      # column_name: CustomType
+      # [obsolete] column_type: !bigint
+      if descriptor =~ /(\[obsolete\])?\s*(\w+):\s*(!?\w+\??(?:\s(\d+))?)/
+        obsolete = $1 != nil
+        name = $2
+        type = Type.parse($3)
+        Column.new(name, type, descriptions, obsolete)
+      else
+        raise "Could not parse column '#{descriptor}'"
+      end
+    end
+
+    def strdef
+      if type.name == 'string' && type.str_length
+        "#{name}: !#{type.name} #{type.str_length}"
+      elsif !type.custom?
+        "#{name}: !#{type.name}"
+      else
+        "#{name}: #{type.name}"
+      end
+    end
+
+    def dump
+      {'name' => name, 'type' => type.dump, 'descriptions' => descriptions}
+    end
+  end
+
+  class Type
+    AVAILABLE_BUILTIN_TYPES = %w(
+      smallint
+      integer
+      bigint
+      real
+      double
+      boolean
+      string
+      date
+      timestamp
+      timestamptz
+    )
+    # @return String
+    attr_reader :name
+    # Length for built-in String types
+    # This parameter returns nil when type is not String
+    # @return Integer?
+    attr_reader :str_length
+
+    def initialize(name, optional, custom, str_length = nil)
+      @name = name
+      @is_optional = optional
+      @is_custom = custom
+      @str_length = str_length
+      unless custom
+        raise "Invalid built-in type #{name}" unless AVAILABLE_BUILTIN_TYPES.include?(name)
+      end
+    end
+
+    # @return Boolean
+    def optional?
+      @is_optional
+    end
+
+    # Boolean value which indicates whether this type is user defined types or not.
+    # @return Boolean
+    def custom?
+      @is_custom
+    end
+
+    def self.parse(descriptor)
+      type_name, str_length = descriptor.split
+      is_optional = type_name.end_with?('?')
+      type_without_optional = type_name.gsub(/\?$/, '')
+      custom = !type_name.start_with?('!')
+      if custom
+        raise "Doesn't support str_length for custom types #{type_name}" unless str_length.nil?
+        Type.new(type_without_optional, is_optional, custom, nil)
+      else
+        built_in_type_name = type_without_optional.gsub(/^!/, '')
+        raise "type_name '#{built_in_type_name}' is not allowed" unless AVAILABLE_BUILTIN_TYPES.include?(built_in_type_name)
+        raise "length for '#{built_in_type_name}' is not supported" if built_in_type_name != 'string' && !str_length.nil?
+
+        Type.new(built_in_type_name, is_optional, custom, str_length&.to_i)
+      end
+    end
+
+    def dump
+      {'name' => name, 'optional' => optional?, 'is_custom' => custom?, 'str_length' => str_length}
+    end
+  end
+end

data/lib/daifuku/parser.rb

@@ -0,0 +1,76 @@
+require 'redcarpet'
+require 'nokogiri'
+
+module Daifuku
+
+  class Parser
+    def parse(markdown_str, category_name)
+      html_str = markdown.render(markdown_str)
+      html = Nokogiri::HTML(html_str).css('body')
+
+      top_level_nodes = html.children
+      nodes_per_event = {}
+      common_columns_nodes = []
+      current_event_name = nil
+      top_level_nodes.each do |node|
+        case node.name
+        when 'h2'
+          current_event_name = node.content
+        when 'p', 'ul'
+          if current_event_name
+            nodes_per_event[current_event_name] ||= []
+            nodes_per_event[current_event_name] << node
+          else
+            common_columns_nodes << node
+          end
+        else
+          next
+        end
+      end
+
+      events = nodes_per_event.map { |name_with_annotation, nodes|
+        event = parse_event(name_with_annotation, nodes)
+        [event.name, event]
+      }.to_h
+      common_event = parse_event(nil, common_columns_nodes)
+      descriptions = common_event&.descriptions || []
+      Category.new(category_name, events, descriptions, common_event&.columns || [])
+    end
+
+    private
+
+    def parse_event(event_name, nodes)
+      if event_name_match = event_name&.match(/\s*\[obsolete\]\s*(.*)/)
+        event_name = event_name_match[1]
+        is_obsolete = true
+      else
+        is_obsolete = false
+      end
+      descriptions = nodes.select { |node| node.name == 'p' }
+                         .map { |node| node.content }
+      columns_node = nodes.select { |node| node.name == 'ul' }.first
+      columns = []
+      columns = parse_columns(columns_node) if columns_node
+      Event.new(event_name, columns, descriptions, is_obsolete)
+    end
+
+    def parse_columns(columns_node)
+      descriptor_nodes = columns_node.css('> li')
+      descriptor_nodes.map do |node|
+        descriptor = node.content
+        descriptions = node.css("ul li").map(&:content)
+        column = Column.parse(descriptor, descriptions)
+        raise "Could not parse '#{descriptor}'" unless column
+        column
+      end
+    end
+
+    def markdown
+      @markdown ||= Redcarpet::Markdown.new(renderer, underline: false, emphasis: false)
+    end
+
+    def renderer
+      @renderer ||= Daifuku::Renderer.new
+    end
+  end
+end

data/lib/daifuku/renderer.rb

@@ -0,0 +1,11 @@
+module Daifuku
+  class Renderer < Redcarpet::Render::HTML
+    def underline(text)
+      nil
+    end
+
+    def emphasis(text)
+      nil
+    end
+  end
+end

data/lib/daifuku/strdef_generator.rb

@@ -0,0 +1,196 @@
+module Daifuku
+    # This class is used to generate .strdef file from .md files.
+    # .strdef file is used to define the structure of the log.
+    # https://techlife.cookpad.com/entry/2019/10/18/090000
+    # Since the `.strdef` generation for Redshift Spectrum is for Cookpad's internal reasons,
+    # we initially decided it would be more natural not to include it in the core of daifuku,
+    # but we added it as a reference implementation.
+    class StrdefGenerator
+      class StreamDefinition
+        def self.parse(strdef)
+          columns = {}
+          strdef.each_line do |line|
+            case line
+            when /\A\s*-\s+(\w+):\s*(!domain\s+\w+|!\w+)(?:\s+(\d+))?/
+              name = $1
+              type = $2.sub(/\s+/, ' ').sub(/\A!/, '')
+              length = $3
+              columns[name] = StreamColumn.new(name, type, length&.to_i)
+            end
+          end
+          new(columns)
+        end
+  
+        include Enumerable
+  
+        def initialize(columns)
+          @columns = columns
+        end
+  
+        def each_column(&block)
+          @columns.each_value(&block)
+        end
+  
+        def [](name)
+          @columns[name]
+        end
+      end
+  
+      class StreamColumn
+        def initialize(name, type, length = nil)
+          @name = name
+          @type = type
+          @length = length
+        end
+  
+        attr_reader :name
+        attr_reader :type
+        attr_reader :length
+  
+        def strdef
+          if length
+            "#{name}: !#{type} #{length}"
+          else
+            "#{name}: !#{type}"
+          end
+        end
+      end
+  
+      class LogTable
+        PREDEFINED = %w[id log_id event_category event_name]
+  
+        def self.predefined_column?(name)
+          PREDEFINED.include?(name)
+        end
+  
+        def initialize
+          @columns = {}
+        end
+  
+        def add_common_column(column)
+          @columns[column.name] = LogColumn.new(column.name, column.type, true)
+        end
+  
+        def add_column(column, location = '-')
+          if prev = @columns[column.name]
+            if prev.common?
+              raise "common column is being overwritten: #{column.name}: location=#{location}"
+            end
+            unless column.type.name == prev.type.name
+              raise "column type mismatch: name=#{column.name}, curr=#{column.type.name}, prev=#{prev.type.name}"
+            end
+          end
+          @columns[column.name] = LogColumn.new(column.name, column.type)
+        end
+  
+        def [](name)
+          @columns[name]
+        end
+  
+        def each_column(&block)
+          @columns.each_value(&block)
+        end
+      end
+  
+      class LogColumn
+        def initialize(name, type, is_common = false)
+          @name = name
+          @type = type
+          @is_common = is_common
+        end
+  
+        attr_reader :name
+        attr_reader :type
+  
+        def common?
+          @is_common
+        end
+  
+        def strdef
+          if type.str_length
+            "#{name}: !#{type.name} #{type.str_length}"
+          else
+            "#{name}: !#{type.name}"
+          end
+        end
+      end
+  
+      # @param definition_path A directory path it contains log definitions
+      def initialize(definition_path)
+        @definition_path = definition_path
+      end
+  
+      # Generate whole strdef from current log definitions
+      # @return String
+      def generate
+        table = load_table_definition(@definition_path)
+        generate_strdef(table)
+      end
+  
+      # Generate strdef diff with existing strdef
+      # @param String current_strdef strdef to diff with
+      # @return String diff
+      def diff(current_strdef)
+        table = load_table_definition(@definition_path)
+        generate_strdef_diff(current_strdef, table)
+      end
+  
+      private
+      def load_table_definition(path_prefix)
+        table = LogTable.new
+  
+        categories = Daifuku::Compiler.new.compile(path_prefix)
+  
+        if common = categories.delete('common')
+          common.common_columns.each do |column|
+            table.add_common_column(column)
+          end
+        end
+  
+        categories.each do |category_name, category|
+          category.common_columns.each do |column|
+            table.add_column column, "#{category_name}:common"
+          end
+          category.events.each do |event_name, event|
+            event.columns.each do |column|
+              table.add_column column, category_name
+            end
+          end
+        end
+  
+        table
+      end
+  
+      def generate_strdef(table)
+        lines = ["columns:"]
+        table.each_column do |column|
+          lines << "  - #{column.strdef}"
+        end
+        lines.join("\n") + "\n"
+      end
+  
+      def generate_strdef_diff(strdef, table)
+        strdef = StreamDefinition.parse(strdef)
+  
+        lines = []
+        found = false
+        table.each_column do |column|
+          unless strdef[column.name]
+            lines << "- #{column.strdef}"
+            found = true
+          end
+        end
+  
+        unless found
+          $stderr.puts %Q(INFO: no new column added)
+        end
+  
+        strdef.each_column do |column|
+          if not table[column.name] and not LogTable.predefined_column?(column.name)
+            $stderr.puts %Q(INFO: column "#{column.name}" is no longer generated)
+          end
+        end
+        lines.join("\n") + "\n"
+      end
+    end
+  end