chronicle-shell 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd121be7bcf453873b98d815bd22b5ef2a685b2e5752d88982ab435ded4e9a1d
-  data.tar.gz: f175ee27114f9b02344387e06a19122f400c37291e67721ad7aecfb32dc77016
+  metadata.gz: cba457d01ec6e7ee3719b80d980ee171797eabef20c7aaadc751e4e985f96f73
+  data.tar.gz: 8ec830dcd00de94eb7d9e9ff4a0942d3cc49f8975a88a792be4ad2be756630a4
 SHA512:
-  metadata.gz: 4e652cbf303a7f5407d3d1bf67c688932836551760e5966a194ee75a2944957a958f3500e371cd116ad8050d798f8667fe24ad1eb01ad1425adf1d6716b909ab
-  data.tar.gz: 434efb8f969f5c28a6e49a89ab88a8193d9dd195365bd7be1d08d83b3cee1c3f8752c71b5e32604dd3bf5a15f49a0f485090d77b0832561f2d19219d0fa34d04
+  metadata.gz: f8f50a8c830a4687d9769d04d9a074f68feafb1ba9a07ac03dd2e4cc3e8794842436d2b968cc25cce91c41a555634ee5bf99a2bfcc5f70c37d306a08ce7108e0
+  data.tar.gz: 0ddbb6f0cb224a324a8b2150ca931e8f226b572b2e4a8fcb29df9616921a08c1b3f208c15737978e655ae3514a489ee234ce20378b973be37991e44a59ea427e

data/.rubocop.yml

@@ -0,0 +1,2 @@
+inherit_gem:
+  chronicle-core: .rubocop-plugin.yml

data/Gemfile

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 # Specify your gem's dependencies in chronicle-shell.gemspec
 gemspec
 
-gem "rake", "~> 13.0"
+gem 'rake', '~> 13.0'

data/README.md

@@ -3,13 +3,6 @@
 
 Shell importer for [chronicle-etl](https://github.com/chronicle-app/chronicle-etl)
 
-## Available Connectors
-### Extractors
-- `shell:history` - Extract shell history from bash or zsh
-
-### Transformers
-- `shell:history` - Turn a shell command into Chronicle Schema
-
 ## Usage and examples
 
 ```bash
@@ -17,9 +10,13 @@ Shell importer for [chronicle-etl](https://github.com/chronicle-app/chronicle-et
 gem install chronicle-etl
 chronicle-etl plugins:install shell
 
-# output commands since Feb 7 as json
-chronicle-etl --extractor shell:history --transformer shell:history --since "2022-02-07" --loader json
+# output commands since 2 weeks ago
+$ chronicle-etl --extractor shell:command --schema chronicle --since 2w --loader json
 
 # Show recent commands sorted by frequency of use
-chronicle-etl --extractor shell:history --limit 500 --fields command --silent | sort | uniq -c | sort -nr
+$ chronicle-etl --extractor shell:command --loader table --limit 500 --fields command --silent | sort | uniq -c | sort -nr
 ```
+
+## Available Connectors
+### Extractors
+- `shell:command` - Extract shell history from bash or zsh

data/Rakefile

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
 task default: %i[]

data/chronicle-shell.gemspec

@@ -1,24 +1,23 @@
 # frozen_string_literal: true
 
-require_relative "lib/chronicle/shell/version"
+require_relative 'lib/chronicle/shell/version'
 
 Gem::Specification.new do |spec|
-  spec.name = "chronicle-shell"
+  spec.name = 'chronicle-shell'
   spec.version = Chronicle::Shell::VERSION
-  spec.authors = ["Andrew Louis"]
-  spec.email = ["andrew@hyfen.net"]
+  spec.authors = ['Andrew Louis']
+  spec.email = ['andrew@hyfen.net']
 
-  spec.summary = "Shell connectors for Chronicle-ETL"
-  spec.description = "Extract shell command history from Bash or Zsh"
-  spec.homepage = "https://github.com/chronicle-app/chronicle-shell"
-  spec.required_ruby_version = ">= 2.7.0"
-  spec.license = "MIT"
+  spec.summary = 'Shell connectors for Chronicle-ETL'
+  spec.description = 'Extract shell command history from Bash or Zsh'
+  spec.homepage = 'https://github.com/chronicle-app/chronicle-shell'
+  spec.license = 'MIT'
 
-  spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org'
 
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/chronicle-app/chronicle-shell"
-  spec.metadata["changelog_uri"] = "https://github.com/chronicle-app/chronicle-shell/releases"
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/chronicle-app/chronicle-shell'
+  spec.metadata['changelog_uri'] = 'https://github.com/chronicle-app/chronicle-shell/releases'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -27,9 +26,17 @@ Gem::Specification.new do |spec|
       (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
     end
   end
-  spec.bindir = "exe"
+  spec.bindir = 'exe'
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 3.1'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
-  spec.add_dependency "chronicle-etl", "~> 0.4.3"
+  spec.add_dependency 'chronicle-core', '~> 0.3'
+  spec.add_dependency 'chronicle-etl', '~> 0.6'
+
+  spec.add_development_dependency 'bundler', '~> 2.3'
+  spec.add_development_dependency 'pry-byebug', '~> 3.10'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rubocop', '~> 1.63'
 end

data/lib/chronicle/shell/shell_history_extractor.rb

@@ -4,9 +4,10 @@ module Chronicle
   module Shell
     class ShellHistoryExtractor < Chronicle::ETL::Extractor
       register_connector do |r|
-        r.provider = 'shell'
+        r.source = :shell
+        r.type = :command
+        r.strategy = :local
         r.description = 'shell command history'
-        r.identifier = 'history'
       end
 
       setting :input
@@ -25,11 +26,11 @@ module Chronicle
       def extract
         @commands.each do |command|
           meta = {
-            username: username,
-            hostname: hostname,
+            username:,
+            hostname:,
             shell_name: @config.shell
           }
-          yield Chronicle::ETL::Extraction.new(data: command, meta: meta)
+          yield build_extraction(data: command, meta:)
         end
       end
 
@@ -42,11 +43,11 @@ module Chronicle
       end
 
       def history_filename_default_bash
-        File.join(Dir.home, ".bash_history")
+        File.join(Dir.home, '.bash_history')
       end
 
       def history_filename_default_zsh
-        File.join(Dir.home, ".zsh_history")
+        File.join(Dir.home, '.zsh_history')
       end
 
       def username
@@ -63,16 +64,17 @@ module Chronicle
       def load_commands
         commands = []
 
-        loader = "load_commands_from_#{@config.shell}"
-        __send__(loader) do |command|
-          next if @config.since && command[:timestamp] < @config.since
-          next if @config.until && command[:timestamp] > @config.until
-
-          if block_given?
-            yield command
-          else
-            commands << command
+        case @config.shell.to_sym
+        when :bash
+          load_commands_from_bash do |command|
+            process_command(command, commands)
+          end
+        when :zsh
+          load_commands_from_zsh do |command|
+            process_command(command, commands)
           end
+        else
+          raise "Unknown loader: #{@config.shell}"
         end
 
         commands = commands.first(@config.limit) if @config.limit
@@ -80,13 +82,24 @@ module Chronicle
         commands
       end
 
+      def process_command(command, commands)
+        return if @config.since && command[:timestamp] < @config.since
+        return if @config.until && command[:timestamp] > @config.until
+
+        if block_given?
+          yield command
+        else
+          commands << command
+        end
+      end
+
       def load_commands_from_bash
         command = nil
         File.readlines(history_filename).reverse_each do |line|
           timestamp_line = line.scrub.match(BASH_TIMESTAMP_REGEX)
           if timestamp_line && command
             timestamp = Time.at(timestamp_line[:timestamp].to_i)
-            command = { timestamp: timestamp, command: command }
+            command = { timestamp:, command: }
             yield command
           else
             command = line.scrub.strip

data/lib/chronicle/shell/shell_history_transformer.rb

@@ -1,59 +1,64 @@
 require 'chronicle/etl'
+require 'chronicle/models'
 
 module Chronicle
-  module Shell 
+  module Shell
     class ShellHistoryTransformer < Chronicle::ETL::Transformer
       register_connector do |r|
-        r.provider = 'shell'
+        r.source = :shell
+        r.type = :command
+        r.strategy = :local
         r.description = 'a shell command'
-        r.identifier = 'history'
+        r.from_schema = :extraction
+        r.to_schema = :chronicle
       end
 
-      def transform
-        @command = @extraction.data
-        build_commanded
-      end
-
-      def id
-      end
+      def transform(record)
+        username = record.extraction.meta[:username]
+        hostname = record.extraction.meta[:hostname]
+        shell_name = record.extraction.meta[:shell_name]
+        timestamp = record.data[:timestamp]
+        command = record.data[:command]
 
-      def timestamp
-        @command[:timestamp]
+        build_command(username:, hostname:, command:, shell_name:,
+          timestamp:)
       end
 
       private
 
-      def build_commanded
-        record = ::Chronicle::ETL::Models::Activity.new
-        record.verb = 'commanded'
-        record.end_at = timestamp
-        record.provider = @extraction.meta[:shell_name]
-        record.dedupe_on << [:verb, :end_at, :provider]
-        record.involved = build_command
-        record.actor = build_actor
-        record
+      def build_command(username:, hostname:, command:, shell_name:, timestamp:)
+        Chronicle::Models::ControlAction.new do |r|
+          r.source = shell_name
+          r.result = build_command_result(command)
+          r.agent = build_agent(username, hostname)
+          r.end_time = timestamp
+          # r.object = build_host
+          r.dedupe_on << %i[source type end_time]
+        end
       end
 
-      def build_command
-        record = ::Chronicle::ETL::Models::Entity.new
-        record.represents = 'command'
-        record.provider = @extraction.meta[:shell_name]
-        record.body = @command[:command]
-        record.dedupe_on << [:body, :provider, :represents]
-        record
+      def build_command_result(text)
+        Chronicle::Models::ComputerCommand.new do |r|
+          r.text = text
+          r.source = 'system'
+          r.dedupe_on << %i[source text type]
+        end
       end
 
-      def build_actor
-        record = ::Chronicle::ETL::Models::Entity.new
-        record.represents = 'identity'
-        record.provider = 'system'
-        record.slug = build_user_slug
-        record.dedupe_on << [:represents, :provider, :slug]
-        record
+      def build_agent(username, hostname)
+        Chronicle::Models::Person.new do |r|
+          r.source = 'system'
+          r.slug = build_user_slug(username, hostname)
+          r.dedupe_on << %i[source slug type]
+        end
       end
 
-      def build_user_slug
-        "#{@extraction.meta[:username]}@#{@extraction.meta[:hostname]}"
+      # TODO: implement this.
+      # TODO: figure out how to represent the host in schema
+      def build_host(hostname); end
+
+      def build_user_slug(username, hostname)
+        "#{username}@#{hostname}"
       end
     end
   end

data/lib/chronicle/shell/version.rb

@@ -2,6 +2,6 @@
 
 module Chronicle
   module Shell
-    VERSION = "0.2.3"
+    VERSION = '0.3.0'
   end
 end

data/lib/chronicle/shell.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative "shell/version"
-require_relative "shell/shell_history_extractor"
-require_relative "shell/shell_history_transformer"
+require_relative 'shell/version'
+require_relative 'shell/shell_history_extractor'
+require_relative 'shell/shell_history_transformer'
 
 module Chronicle
   module Shell

metadata

@@ -1,29 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-shell
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-16 00:00:00.000000000 Z
+date: 2024-05-01 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: chronicle-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: chronicle-etl
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.3
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.3
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.63'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.63'
 description: Extract shell command history from Bash or Zsh
 email:
 - andrew@hyfen.net
@@ -31,6 +101,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".rubocop.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -49,6 +120,7 @@ metadata:
   homepage_uri: https://github.com/chronicle-app/chronicle-shell
   source_code_uri: https://github.com/chronicle-app/chronicle-shell
   changelog_uri: https://github.com/chronicle-app/chronicle-shell/releases
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -57,14 +129,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Shell connectors for Chronicle-ETL