bigsister 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 801cec260afc76e68534a6eb2a927af3ded0e4a3127b87288cc7129ca11fb643
-  data.tar.gz: ee112f0f35496f9c96271e57774c9cbb814a704d2d97f5bba7f59b156fbf99b3
+  metadata.gz: e856359282c49195b70bf99957e8a51b15829dd04db3e60585cfeff07e07f7bd
+  data.tar.gz: 2a0a5e21088f29e54bb2677c13d05a75d43a155d244072cf5a6d3ff667006eb5
 SHA512:
-  metadata.gz: cab071a5a1faf95f92221df55e7bcef07b75ee8220d241572a4fc624a2b150d2260882802ed2d1e8032e0ac54974ee72d18bd4f188968080b5d70d3b8f7699c0
-  data.tar.gz: 82720dfdfcb3d29abbd233f8ad0c2a51a6b2e0e3b12b289d485366c3a206baf6bc9712f01cdbb632fb747f1eabb88f8d5852aab25d0c73b214228ab39b2ae0f4
+  metadata.gz: 78f223af17f62af998f0ce1589472a6d642ca8eb87e22e92ebc44c6f129063e4cfaa8d932cb2e59dad0beba01d0f8f813abd70f8b3c6b507b08e80bd03cf7f8f
+  data.tar.gz: 920e554d8c4796f8113f80e688a6ea21e6461f07eda99bda0e12995cbc4e7ada5b8e9b425f3d8ff45895fe92757fb6f5735f349144f1a284c51541f1e84809b3

data/.gitignore

@@ -11,3 +11,4 @@
 .rspec_status
 
 Gemfile.lock
+.bigsister.yml

data/GHANGELOG.md

@@ -4,10 +4,16 @@ All notable changes to BigSister will be documented in this file.
 
 ## [Unreleased]
 
+## [0.1.2] - Nov 11, 2019
+### Added
+* CSV reporter
+* Complete monitoring and report generation
+
 ## [0.1.1] - Nov 8, 2019
 ### Added
 * Basic Application object
 * CLI tool
 
-[Unreleased]: https://github.com/paulholden2/bigsister/compare/0.1.1...HEAD
+[Unreleased]: https://github.com/paulholden2/bigsister/compare/0.1.2...HEAD
 [0.1.1]: https://github.com/paulholden2/bigsister/releases/tag/0.1.1
+[0.1.2]: https://github.com/paulholden2/bigsister/releases/tag/0.1.2

data/README.md

@@ -34,12 +34,8 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/bigsister. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/paulholden2/bigsister. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
-
-## Code of Conduct
-
-Everyone interacting in the BigSister project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/bigsister/blob/master/CODE_OF_CONDUCT.md).

data/bigsister.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "springcm-sdk", "~> 0.3"
+  spec.add_dependency "springcm-sdk", "~> 0.3.2"
   spec.add_dependency "os", "~> 1.0"
 
   spec.add_development_dependency "bundler", "~> 2.0"

data/lib/bigsister.rb

@@ -20,4 +20,7 @@ module BigSister
       super("No configuration files found. Looked in: #{checked_paths.join(', ')}")
     end
   end
+
+  class InvalidConfiguration < Error
+  end
 end

data/lib/bigsister/application.rb

@@ -15,6 +15,21 @@ module BigSister
     end
 
     def run
+      config.reporters.each { |reporter|
+        config.monitors.each { |monitor|
+          if reporter.log_files?
+            monitor.files.each { |file|
+              reporter.log_file(file)
+            }
+          end
+          if reporter.log_directories?
+            monitor.directories.each { |directory|
+              reporter.log_directory(directory)
+            }
+          end
+        }
+        reporter.render
+      }
     end
 
     private

data/lib/bigsister/configuration.rb

@@ -1,17 +1,21 @@
 require "bigsister/monitors/local"
 require "bigsister/monitors/springcm"
+require "bigsister/reporters/csv_reporter"
 
 module BigSister
   class Configuration
     MONITOR_TYPES = %w(local springcm).freeze
+    REPORTER_TYPES = %w(csv).freeze
 
-    attr_reader :monitors
+    attr_reader :monitors, :reporters
 
     def initialize(data)
       @monitors = []
+      @reporters = []
       @config_errors = []
 
       load_monitoring(data)
+      load_reporting(data)
     end
 
     def valid?
@@ -22,9 +26,6 @@ module BigSister
 
     def load_monitoring(data)
       @monitoring = data.fetch("monitor", [])
-      if @monitoring.empty?
-        config_error("No monitoring locations configured")
-      end
       @monitoring.each_with_index { |location, i|
         # Validate monitoring type
         type = location.fetch("type", nil)
@@ -46,6 +47,27 @@ module BigSister
       end
     end
 
+    def load_reporting(data)
+      @reporting = data.fetch("report", [])
+      @reporting.each_with_index { |reporter, i|
+        # Validate reporting type
+        type = reporter.fetch("type", nil)
+        if type.nil?
+          config_error("No report type specified for reporter #{i}")
+        elsif !REPORTER_TYPES.include?(type)
+          config_error("Invalid report type specifeid for reporter #{i}")
+        end
+        @reporters.push(load_reporter(type, reporter, i))
+        @reporters = @reporters.reject(&:nil?)
+      }
+    end
+
+    def load_reporter(type, reporter, i)
+      if type == "csv"
+        BigSister::CsvReporter.new(reporter, i)
+      end
+    end
+
     def config_error(message)
       @config_errors.push(message)
     end

data/lib/bigsister/directory_info.rb

@@ -0,0 +1,19 @@
+module BigSister
+  class DirectoryInfo
+    PROPERTIES = %w(name file_count directory_count).map(&:to_sym).freeze
+
+    PROPERTIES.each { |prop|
+      attr_reader prop
+    }
+
+    def initialize(data = {})
+      @name = data[:name]
+      @file_count = data[:file_count]
+      @directory_count = data[:directory_count]
+    end
+
+    def type
+      "directory"
+    end
+  end
+end

data/lib/bigsister/file_info.rb

@@ -0,0 +1,18 @@
+module BigSister
+  class FileInfo
+    PROPERTIES = %w(name file_size).map(&:to_sym).freeze
+
+    PROPERTIES.each { |prop|
+      attr_reader prop
+    }
+
+    def initialize(data = {})
+      @name = data[:name]
+      @file_size = data[:file_size]
+    end
+
+    def type
+      "file"
+    end
+  end
+end

data/lib/bigsister/monitor.rb

@@ -4,11 +4,5 @@ module BigSister
       @location = location
       @id = id
     end
-
-    def summary
-    end
-
-    def detail
-    end
   end
 end

data/lib/bigsister/monitors/local.rb

@@ -1,6 +1,45 @@
 require "bigsister/monitor"
+require "bigsister/file_info"
+require "bigsister/directory_info"
 
 module BigSister
   class LocalMonitor < BigSister::Monitor
+    def initialize(config, i)
+      @config = config
+      @id = i
+      @path = config.fetch("path", nil)
+      super(config, i)
+    end
+
+    def files
+      res = Dir.entries(@path).reject { |file|
+        File.directory?(File.join(@path, file))
+      }.map { |file|
+        file_path = File.join(@path, file)
+        FileInfo.new(
+          name: file,
+          path: file_path,
+          file_size: File.size(file_path)
+        )
+      }
+      res
+    end
+
+    def directories
+      res = Dir.entries(@path).select { |file|
+        File.directory?(File.join(@path, file))
+      }.map { |dir|
+        dir_path = File.join(@path, dir)
+        file_count = Dir.entries(dir_path).reject { |file| File.directory?(File.join(dir_path, file)) }.size
+        directory_count = Dir.entries(dir_path).select { |file| File.directory?(File.join(dir_path, file)) }.size
+        DirectoryInfo.new(
+          name: dir,
+          path: dir_path,
+          file_count: file_count,
+          directory_count: directory_count
+        )
+      }
+      res
+    end
   end
 end

data/lib/bigsister/monitors/springcm.rb

@@ -1,6 +1,70 @@
 require "bigsister/monitor"
+require "bigsister/file_info"
+require "bigsister/directory_info"
+require "springcm-sdk"
 
 module BigSister
   class SpringcmMonitor < BigSister::Monitor
+    def initialize(config, i)
+      @client_id = config.fetch("client_id", nil)
+      @client_secret = config.fetch("client_secret", nil)
+      @data_center = config.fetch("data_center", nil)
+      @path = config.fetch("path", nil)
+      validate_config!
+      @client = Springcm::Client.new(@data_center, @client_id, @client_secret)
+      @client.connect!
+      @folder = @client.folder(path: @path)
+      super(config, i)
+    end
+
+    def files
+      docs = []
+      list = @folder.documents(offset: 0, limit: 20)
+      while !list.nil?
+        docs = docs + list.items
+        list = list.next
+      end
+      res = docs.map { |document|
+        FileInfo.new(
+          name: document.name,
+          file_size: document.native_file_size
+        )
+      }
+      res
+    end
+
+    def directories
+      dirs = []
+      list = @folder.folders(offset: 0, limit: 20)
+      while !list.nil?
+        dirs = dirs + list.items
+        list = list.next
+      end
+      res = dirs.map { |dir|
+        DirectoryInfo.new(
+          name: dir.name,
+          file_count: dir.documents.total,
+          directory_count: dir.folders.total
+        )
+      }
+      res
+    end
+
+    private
+
+    def validate_config!
+      if @client_id.nil?
+        raise BigSister::InvalidConfiguration.new("SpringCM client_id required.")
+      end
+      if @client_secret.nil?
+        raise BigSister::InvalidConfiguration.new("SpringCM client_secret is required.")
+      end
+      if @client_secret.nil?
+        raise BigSister::InvalidConfiguration.new("SpringCM data_center is required.")
+      end
+      if @path.nil?
+        raise BigSister::InvalidConfiguration.new("SpringCM path is required.")
+      end
+    end
   end
 end

data/lib/bigsister/reporter.rb

@@ -0,0 +1,136 @@
+module BigSister
+  class Reporter
+    def initialize(schema, id)
+      @id = id
+      @schema = schema
+      @files = []
+      @directories = []
+      if !@schema.key?("format")
+        raise BigSister::InvalidConfiguration.new("Reporter format is required")
+      elsif !%w(summary detail).include?(@schema.fetch("format"))
+        raise BigSister::InvalidConfiguration.new("Reporter format must be one of: summary, detail")
+      else
+        @format = @schema.fetch("format")
+      end
+      @columns = schema.fetch("columns", [])
+      @columns.each { |column|
+        validate_column!(column)
+      }
+    end
+
+    def log_file(path)
+      @files.push(path)
+    end
+
+    def log_directory(path)
+      @directories.push(path)
+    end
+
+    def summary?
+      return @format == "summary"
+    end
+
+    def detail?
+      return !summary?
+    end
+
+    def log_files?
+      return @schema.fetch("include", []).include?("files")
+    end
+
+    def log_directories?
+      return @schema.fetch("include", []).include?("directories")
+    end
+
+    protected
+
+    def validate_column!(column)
+      if !column.key?("type")
+        raise BigSister::InvalidConfiguration.new("Column type is required.")
+      end
+      allowed_types = %w(timestamp literal file_count directory_count)
+      if detail?
+        allowed_types += %w(file_size type name)
+      end
+      type = column.fetch("type")
+      if !allowed_types.include?(type)
+        list = allowed_types.join(", ")
+        raise BigSister::InvalidConfiguration.new("Column type for #{@format} must be one of: #{list}")
+      end
+    end
+
+    def transform_file(file, column)
+      type = column["type"]
+      if type == "timestamp"
+        DateTime.now.strftime("%FT%T%:z")
+      elsif type == "name"
+        file.name
+      elsif type == "type"
+        file.type
+      elsif type == "file_size"
+        file.file_size
+      elsif type == "literal"
+        @schema.fetch("value", nil)
+      end
+    end
+
+    def transform_directory(directory, column)
+      type = column["type"]
+      if type == "timestamp"
+        current_timestamp
+      elsif type == "name"
+        directory.name
+      elsif type == "type"
+        directory.type
+      elsif type == "file_count"
+        directory.file_count
+      elsif type == "directory_count"
+        directory.directory_count
+      elsif type == "literal"
+        @schema.fetch("value", nil)
+      end
+    end
+
+    def file_rows
+      @files.map { |file|
+        @columns.map { |column|
+          transform_file(file, column)
+        }
+      }
+    end
+
+    def directory_rows
+      @directories.map { |dir|
+        @columns.map { |column|
+          transform_directory(dir, column)
+        }
+      }
+    end
+
+    def current_timestamp
+      DateTime.now.strftime("%FT%T%:z")
+    end
+
+    def detail
+      rows = []
+      rows += file_rows if log_files?
+      rows += directory_rows if log_directories?
+      rows
+    end
+
+    def summary
+      file_count = file_rows.size
+      directory_count = directory_rows.size
+      @columns.map { |column|
+        type = column["type"]
+        if type == "timestamp"
+          current_timestamp
+        elsif type == "file_count"
+          file_count
+        elsif type == "directory_count"
+          directory_count
+        end
+      }
+    end
+  end
+end

data/lib/bigsister/reporters/csv_reporter.rb

@@ -0,0 +1,44 @@
+require "bigsister/reporter"
+require "csv"
+
+module BigSister
+  class CsvReporter < BigSister::Reporter
+    def initialize(schema, i)
+      super(schema, i)
+      if !schema.key?("path")
+        raise BigSister::InvalidConfiguration.new("CSV Reporter path is required.")
+      else
+        @outfile_path = schema.fetch("path")
+      end
+    end
+
+    def csv
+      if summary?
+        summary
+      else
+        detail
+      end
+    end
+
+    def render
+      File.open(@outfile_path, "w") { |file|
+        file.write(csv)
+      }
+    end
+
+    protected
+
+    def headers
+      @columns.map { |column| column["title"] }.to_csv(row_sep: nil)
+    end
+
+    def detail
+      rows = super.map { |row| row.to_csv(row_sep: nil) }
+      ([headers] + rows).join("\n")
+    end
+
+    def summary
+      [headers, super.to_csv(row_sep: nil)].join("\n")
+    end
+  end
+end

data/lib/bigsister/version.rb

@@ -1,3 +1,3 @@
 module BigSister
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bigsister
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Paul Holden
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-11-08 00:00:00.000000000 Z
+date: 2019-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: springcm-sdk
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 0.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 0.3.2
 - !ruby/object:Gem::Dependency
   name: os
   requirement: !ruby/object:Gem::Requirement
@@ -102,7 +102,6 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".bigsister.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -118,9 +117,13 @@ files:
 - lib/bigsister.rb
 - lib/bigsister/application.rb
 - lib/bigsister/configuration.rb
+- lib/bigsister/directory_info.rb
+- lib/bigsister/file_info.rb
 - lib/bigsister/monitor.rb
 - lib/bigsister/monitors/local.rb
 - lib/bigsister/monitors/springcm.rb
+- lib/bigsister/reporter.rb
+- lib/bigsister/reporters/csv_reporter.rb
 - lib/bigsister/version.rb
 homepage: https://github.com/paulholden2/bigsister
 licenses:

data/.bigsister.yml

@@ -1,49 +0,0 @@
-monitor:
-  - type: local
-    path: /Users/paul
-  - type: springcm
-    path: /Human Resources Records
-    credentials:
-      client_id: client_id
-      client_secret: client_secret
-      data_center: data_center
-generate:
-  - type: csv
-    path: ./detail.csv
-    format: detail
-    mode: write
-    include:
-      - directories
-      - files
-    columns:
-      - title: Timestamp
-        type: timestamp
-      - title: Path
-        type: path
-      - title: File Size
-        type: size
-      - title: File Type
-        type: type
-      - title: Created
-        type: created_at
-      - title: Modified
-        type: modified_at
-      - title: Customer
-        type: literal
-        value: KHSD
-  - type: csv
-    path: ./summary.csv
-    format: summary
-    mode: append
-    include:
-      - directories
-      - files
-    columns:
-      - title: Timestamp
-        type: timestamp
-      - title: File Count
-        type: file_count
-      - title: Directory Count
-        type: directory_count
-      - title: File Size
-        type: file_size