fb_bookworm 0.0.1

data/lib/bookworm/keys.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Bookworm
+  # These keys are how we generalize and generate a lot of code within Bookworm
+  #
+  # VALUES
+  # metakey: A metakey isn't crawled, but is useful as a place to store information
+  # plural: The pluralized form of the key - typically used in method creation
+  # source_dirs: Specify which array of source directories to crawl
+  # glob_pattern: Specify the glob pattern for which files to crawl in source directories
+  # dont_init_kb_key: Don't initialize the keys on knowledge base creation
+  # determine_cookbook_name: Determines the cookbook name from the given path
+  # path_name_regex: A regex with a capture to determine the prettified name of the file
+  BOOKWORM_KEYS = {
+    'cookbook' => {
+      'metakey' => true,
+      'dont_init_kb_key' => true,
+    },
+    'role' => {
+      'source_dirs' => 'role_dirs',
+      'glob_pattern' => '*.rb',
+      'path_name_regex' => '([\w-]+)\.rb',
+    },
+    'metadatarb' => {
+      'glob_pattern' => '*/metadata.rb',
+      'determine_cookbook_name' => true,
+      'path_name_regex' => '(metadata\.rb)',
+    },
+    'recipe' => {
+      'determine_cookbook_name' => true,
+      'path_name_regex' => 'recipes/(.*)\.rb',
+    },
+    'attribute' => {
+      'determine_cookbook_name' => true,
+      'path_name_regex' => 'attributes/(.*)\.rb',
+    },
+    'library' => {
+      'plural' => 'libraries', # <-- the troublemaker that prompted keys first ;-)
+      'determine_cookbook_name' => true,
+      'path_name_regex' => 'libraries\/(.*)\.rb',
+    },
+    'resource' => {
+      'determine_cookbook_name' => true,
+      'path_name_regex' => 'resources/(.*)\.rb',
+    },
+    'provider' => {
+      'determine_cookbook_name' => true,
+      'path_name_regex' => 'providers/(.*)\.rb',
+    },
+  }.freeze
+
+  # Set defaults
+  BOOKWORM_KEYS.each do |k, v|
+    BOOKWORM_KEYS[k]['determine_cookbook_name'] ||= false
+    BOOKWORM_KEYS[k]['plural'] ||= "#{k}s"
+    BOOKWORM_KEYS[k]['source_dirs'] ||= 'cookbook_dirs'
+    BOOKWORM_KEYS[k]['glob_pattern'] ||= "*/#{v['plural']}/*.rb"
+  end
+
+  BOOKWORM_KEYS.freeze
+end

data/lib/bookworm/knowledge_base.rb

@@ -0,0 +1,117 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'pathname'
+module Bookworm
+  module KnowledgeBaseBackends
+    # The SimpleHash backend stores the KnowledgeBase information as ... a
+    # simple hash. No concurrency guarantees, caching, etc. Each bookworm run
+    # will run all the rules, every time.
+    module SimpleHash
+      def init_hooks
+        @kb_internal_hash = {}
+      end
+
+      def [](key)
+        @kb_internal_hash[key]
+      end
+
+      def []=(key, value)
+        @kb_internal_hash[key] = value
+      end
+    end
+  end
+
+  # The KnowledgeBase is a backend-agnostic way of storing and querying
+  # information that's generated about the files via InferRules.
+  #
+  # We provide indirect access to the information stored by the
+  # knowledge base, so that we'll soon be able to leverage different
+  # backends (for concurrent writes, caching rule output across runs, etc).
+  class KnowledgeBase
+    def initialize(opts)
+      extend Bookworm::KnowledgeBaseBackends::SimpleHash
+
+      init_hooks
+
+      # TODO: Only initialize keys required by rules
+      BOOKWORM_KEYS.each do |k, v|
+        unless v['dont_init_kb_key']
+          if v['determine_cookbook_name']
+            init_key_with_cookbook_name(k, opts[k] || [])
+          else
+            init_key(k, opts[k] || [])
+          end
+        end
+        create_pluralized_getter(k)
+      end
+    end
+
+    def backend_missing
+      fail 'Need to specify a backend for KnowledgeBase class'
+    end
+
+    def [](_key)
+      backend_missing
+    end
+
+    def []=(_key, _value)
+      backend_missing
+    end
+
+    def init_hooks
+      # This is optional for KnowledgeBaseBackends
+    end
+
+    private
+
+    # This creates a method based off the (pluralized) Bookworm key.
+    # Syntactical sugar that can make some rules/reports easier to read.
+    def create_pluralized_getter(key)
+      define_singleton_method(BOOKWORM_KEYS[key]['plural'].to_sym) do
+        self[key]
+      end
+    end
+
+    def init_key(key, files)
+      self[key] = {}
+      path_name_regex = BOOKWORM_KEYS[key]['path_name_regex']
+      files.each do |path, ast|
+        m = path.match(/#{path_name_regex}/)
+        file_name = m[1]
+        self[key][file_name] = { 'path' => path, 'ast' => ast }
+      end
+    end
+
+    # The difference between this method and init_key is that it:
+    # 1. initializes cookbook metakey if it doesn't already exist
+    # 2. instead of using the filename for file key, uses COOKBOOK::FILENAME
+    # where FILENAME has the '.rb' suffix stripped (making it similar to the
+    # include_recipe calling conventions in Chef
+    def init_key_with_cookbook_name(key, files)
+      self[key] = {}
+      self['cookbook'] ||= {}
+      path_name_regex = BOOKWORM_KEYS[key]['path_name_regex']
+      files.each do |path, ast|
+        m = path.match(%r{/?([\w-]+)/#{path_name_regex}})
+        cookbook_name = m[1]
+        file_name = m[2]
+        self['cookbook'][cookbook_name] ||= {}
+        self[key]["#{cookbook_name}::#{file_name}"] =
+          { 'path' => path, 'cookbook' => cookbook_name, 'ast' => ast }
+      end
+    end
+  end
+end

data/lib/bookworm/load_hack.rb

@@ -0,0 +1,21 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO: This is gross, and I seem to recall there was some way to get a gem's
+# library directory, but this should work for now.
+module Bookworm
+  BUILTIN_REPORTS_DIR = "#{__dir__}/reports/".freeze
+  BUILTIN_RULES_DIR = "#{__dir__}/rules/".freeze
+end

data/lib/bookworm/report_builder.rb

@@ -0,0 +1,93 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'json'
+require 'pathname'
+
+module Bookworm
+  class BaseReport
+    class << self
+      {
+        'description' => '',
+        'needs_rules' => [],
+      }.each do |attribute, default_value|
+        instance_variable_set("@#{attribute}".to_sym, default_value)
+        define_method(attribute.to_sym) do |val = nil|
+          instance_variable_set("@#{attribute}", val) unless val.nil?
+          instance_variable_get("@#{attribute}")
+        end
+      end
+    end
+
+    def initialize(knowledge_base)
+      @kb = knowledge_base
+      output
+    end
+
+    def to_plain
+      ''
+    end
+
+    def to_json(*_args)
+      JSON.dump({})
+    end
+
+    def default_output
+      :to_plain
+    end
+
+    def output
+      send(default_output)
+    end
+  end
+
+  # Initialize constant for Bookworm::Reports
+  module Reports; end
+
+  class ReportBuilder
+    def initialize(knowledge_base, report_name)
+      klass = Module.const_get("Bookworm::Reports::#{report_name}")
+      output = klass.new(knowledge_base).output
+
+      puts output
+    end
+  end
+
+  def self.get_report_rules(report_name)
+    klass = Module.const_get("Bookworm::Reports::#{report_name}")
+    klass.needs_rules
+  end
+
+  def self.load_report_class(name, dir: '')
+    f = File.read "#{dir}/#{name.to_sym}.rb"
+    ::Bookworm::Reports.const_set(name.to_sym, ::Class.new(::Bookworm::BaseReport))
+    ::Bookworm::Reports.const_get(name.to_sym).class_eval(f)
+  rescue StandardError
+    raise Bookworm::ClassLoadError
+  end
+
+  def self.load_reports_dir(dir)
+    files = Dir.glob("#{dir}/*.rb")
+    files.each do |f|
+      name = Pathname(f).basename.to_s.gsub('.rb', '')
+      begin
+        Bookworm.load_report_class name, :dir => dir
+      rescue Bookworm::ClassLoadError
+        puts "Unable to load report #{f}"
+        exit(false)
+      end
+    end
+  end
+end

data/lib/bookworm/reports/AllReferencedRecipes.rb

@@ -0,0 +1,35 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Determines all recipes that are directly referenced by all roles'
+needs_rules ['RoleRunListRecipes', 'IncludeRecipeLiterals']
+
+def to_a
+  buffer = Set.new
+  @kb.roles.each do |_, metadata|
+    metadata['RoleRunListRecipes'].each do |role|
+      buffer << role
+    end
+  end
+  @kb.recipes.each do |_, metadata|
+    metadata['IncludeRecipeLiterals'].each do |recipe|
+      buffer << recipe
+    end
+  end
+  buffer.sort.to_a
+end
+
+def output
+  to_a
+end

data/lib/bookworm/reports/AllRoleDescriptions.rb

@@ -0,0 +1,25 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Basic report to show extraction of description from roles'
+needs_rules ['RoleName', 'RoleDescription']
+
+def to_plain
+  buffer = ''
+  @kb.roles.sort_by { |k, _| k }.each do |name, metadata|
+    # buffer << name
+    buffer << "role: #{name} desc: #{metadata['RoleDescription']}\n"
+  end
+  buffer
+end

data/lib/bookworm/reports/CookbookDependencyDot.rb

@@ -0,0 +1,30 @@
+# Copyright (c) 2023-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Determine cookbook dependencies from cookbook metadata.rb, output to Dot language'
+needs_rules ['ExplicitMetadataDepends']
+
+def to_s
+  cookbook_deps = []
+  @kb.metadatarbs.each do |x, metadata|
+    metadata['ExplicitMetadataDepends'].each do |cb|
+      cookbook_deps << [x.gsub(/:.*/, ''), cb]
+    end
+  end
+  "digraph deps {\n#{cookbook_deps.map { |arr| arr.join('->') }.join("\n")}\n}"
+end
+
+def output
+  to_s
+end

data/lib/bookworm/reports/DynamicRecipeInclusion.rb

@@ -0,0 +1,29 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Determines all recipes using dynamic recipe inclusion (ie not string literals)'
+needs_rules ['IncludeRecipeDynamic']
+
+def to_a
+  buffer = []
+  @kb.recipes.each do |recipe, metadata|
+    buffer << recipe if metadata['IncludeRecipeDynamic']
+  end
+  buffer.sort!
+  buffer
+end
+
+def output
+  to_a
+end

data/lib/bookworm/reports/LeafCookbooks.rb

@@ -0,0 +1,30 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Determine all leaf cookbooks, where no other cookbook depends on it via metadata.rb'
+needs_rules ['ExplicitMetadataDepends']
+
+def to_a
+  buffer = Set.new
+  @kb.metadatarbs.each do |_, metadata|
+    metadata['ExplicitMetadataDepends'].each do |cb|
+      buffer << cb
+    end
+  end
+  (@kb.cookbooks.keys.to_set - buffer).sort.to_a
+end
+
+def output
+  to_a
+end

data/lib/bookworm/reports/LibraryDefinedModulesAndClassConstants.rb

@@ -0,0 +1,27 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Get modules and classes constants that are explicitly defined in library files'
+needs_rules ['LibraryDefinedClassConstants', 'LibraryDefinedModuleConstants']
+
+def output
+  buffer = ''
+  buffer << "file\tmodules\tconstants\n"
+  @kb.libraries.sort_by { |k, _| k }.each do |name, metadata|
+    # buffer << name
+    buffer << "#{name}:\t#{metadata['LibraryDefinedModuleConstants'].join(',')}" +
+              "\t#{metadata['LibraryDefinedClassConstants'].join(',')}\n"
+  end
+  buffer
+end

data/lib/bookworm/reports/MissingReferencedRecipes.rb

@@ -0,0 +1,61 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Determines all recipes that are directly referenced in roles and recipes' +
+            ' but were not found by the crawler'
+needs_rules ['RoleRunListRecipes', 'IncludeRecipeLiterals']
+
+def to_h
+  known_recipes = Set.new @kb.recipes.keys
+  missing = { 'roles' => {}, 'recipes' => {} }
+  @kb.roles.each do |role, metadata|
+    metadata['RoleRunListRecipes'].each do |recipe|
+      unless known_recipes.include? recipe
+        missing['roles'][role] ||= []
+        missing['roles'][role].append recipe
+      end
+    end
+  end
+  @kb.recipes.each do |kbrecipe, metadata|
+    metadata['IncludeRecipeLiterals'].each do |recipe|
+      unless known_recipes.include? recipe
+        missing['recipes'][kbrecipe] ||= []
+        missing['recipes'][kbrecipe].append recipe
+      end
+    end
+  end
+  missing
+end
+
+def to_plain
+  hsh = to_h
+  buffer = ''
+  if hsh['roles'].empty?
+    buffer << "No missing recipes coming from the roles files\n"
+  else
+    buffer << "Roles:\n"
+    hsh['roles'].each do |role, recipes|
+      buffer << "\t#{role}\t#{recipes.join(', ')}\n"
+    end
+  end
+  if hsh['recipes'].empty?
+    buffer << 'No missing recipes coming from the recipe files'
+  else
+    buffer << "Recipes:\n"
+    hsh['recipes'].each do |kbrecipe, recipes|
+      buffer << "\t#{kbrecipe}\t#{recipes.join(', ')}\n"
+    end
+  end
+  buffer
+end

data/lib/bookworm/reports/NoParsedRuby.rb

@@ -0,0 +1,46 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Ruby files which are empty, comments-only, ' +
+   'or unparseable by RuboCop'
+needs_rules ['NoParsedRuby']
+
+def to_h
+  no_parsed_ruby = {}
+  Bookworm::InferRules::NoParsedRuby.keys.each do |key|
+    plural = BOOKWORM_KEYS[key]['plural']
+    no_parsed_ruby[plural] = []
+    @kb.send(plural.to_sym).each do |k, metadata|
+      no_parsed_ruby[plural].append(k) if metadata['NoParsedRuby']
+    end
+  end
+  no_parsed_ruby
+end
+
+def to_plain
+  hsh = to_h
+  buffer = ''
+  Bookworm::InferRules::NoParsedRuby.keys.each do |key|
+    plural = BOOKWORM_KEYS[key]['plural']
+    if hsh[plural].empty?
+      buffer << "No non-AST #{plural} files\n"
+    else
+      buffer << "#{plural.capitalize}:\n"
+      hsh[plural].sort.each do |keys|
+        buffer << "\t#{keys}\n"
+      end
+    end
+  end
+  buffer
+end

data/lib/bookworm/reports/NotReferencedRecipes.rb

@@ -0,0 +1,37 @@
+# Copyright (c) 2023-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Determines all recipes that are not directly referenced either in roles or recipes'
+needs_rules ['RoleRunListRecipes', 'IncludeRecipeLiterals']
+
+def to_a
+  referenced_recipes = Set.new
+  @kb.roles.each do |_, metadata|
+    metadata['RoleRunListRecipes'].each do |role|
+      referenced_recipes << role
+    end
+  end
+  @kb.recipes.each do |_, metadata|
+    metadata['IncludeRecipeLiterals'].each do |recipe|
+      referenced_recipes << recipe
+    end
+  end
+  all_recipes = Set.new(@kb.recipes.keys)
+  # Any recipes which weren't included via roles or include_recipe are not referenced
+  (all_recipes - referenced_recipes).sort.to_a
+end
+
+def output
+  to_a
+end

data/lib/bookworm/reports/RecipesAssigningConstants.rb

@@ -0,0 +1,27 @@
+# Copyright (c) 2022-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Determine all recipes that are assigning a constant in the recipe file'
+needs_rules ['RecipeConstantAssignments']
+
+def output
+  buffer = ''
+
+  @kb.recipes.each do |name, metadata|
+    unless metadata['RecipeConstantAssignments'].empty?
+      buffer << "#{name} #{metadata['RecipeConstantAssignments'].join(', ')}\n"
+    end
+  end
+  buffer
+end