linkedin-member-data 0.1.0

data/lib/linkedin/member_data/changelog.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require_relative "changelog/page"
+
+module LinkedIn
+  module MemberData
+    # Lazy view over `GET /rest/memberChangeLogs` (last 28 days).
+    # Nothing is fetched until you iterate. Each iteration starts a new walk.
+    #
+    # The cursor is `startTime` = last `processedAt` seen. LinkedIn returns the
+    # cursor event again on the next page, so events already seen are skipped
+    # and iteration stops when a page brings nothing new, or when the last event
+    # has no processedAt (the cursor cannot move).
+    #
+    # Known limit: if more than `count` events share one processedAt, the ones
+    # beyond the first page are not reachable through the cursor.
+    # A higher `count` (max 50) makes this less likely.
+    #
+    # It includes `Enumerable`, so `first`, `to_a`, `lazy`, `select` and the rest work on events.
+    #
+    # @example Print recent events
+    #   client.changelog(since: Time.now - 7 * 86_400).each do |event|
+    #     puts "#{event.method} #{event.resource_name} at #{event.processed_at}"
+    #   end
+    class Changelog
+      include Enumerable
+
+      # API path of the changelog resource.
+      # @return [String]
+      PATH = "/rest/memberChangeLogs"
+
+      # Allowed values for `count`.
+      # @return [Range<Integer>]
+      COUNT_RANGE = (1..50)
+
+      # Default for `count`.
+      # @return [Integer]
+      DEFAULT_COUNT = 10
+
+      # Cursor state of one walk. A new one is built for every iteration.
+      # @api private
+      Walk = Struct.new(:start_time, :seen_ids) do
+        # `page` is an undeduped Page. Returns a Page with only the fresh events
+        # and moves the cursor; nil when nothing is new.
+        def take(page)
+          fresh = fresh_events(page)
+          return if fresh.empty?
+
+          advance(page)
+          Page.new(events: fresh, raw: page.raw)
+        end
+
+        # Events of `page` whose id was not on the previous page.
+        # @api private
+        def fresh_events(page) = page.events.reject { |event| seen_ids.include?(event.id) }
+
+        # Yields each start_time to the block to fetch a Page. Stops at the first
+        # page with nothing new or when the cursor cannot move.
+        def run(yielder)
+          while (fresh_page = take(yield(start_time)))
+            yielder << fresh_page
+            break if start_time.nil?
+          end
+        end
+
+        # Only the previous page is remembered: that is the documented overlap.
+        def advance(page)
+          self.seen_ids = page.events.map(&:id)
+          self.start_time = page.next_start_time
+        end
+      end
+      private_constant :Walk
+
+      # Settings of this view. `since` is the first `processedAt` to fetch, in epoch milliseconds
+      # (`nil` starts at the oldest event). `count` is the number of events per request, from 1 to 50.
+      # @return [Integer, nil] `since` or `count`. `count` is never nil.
+      attr_reader :since, :count
+
+      # @api private
+      # @param connection [Connection] used for every request.
+      # @param since [Time, Date, Integer, nil] first `processedAt` to fetch.
+      #   Integer is epoch milliseconds. A Date is midnight UTC.
+      # @param count [Integer] events per request, from 1 to 50.
+      # @raise [ArgumentError] when `count` is outside 1..50 or `since` has an unsupported type.
+      #   Raised before any request.
+      def initialize(connection, since: nil, count: DEFAULT_COUNT)
+        @count = count
+        ensure_count
+        @connection = connection
+        @since = Util.epoch_ms(since)
+      end
+
+      # Yields every new event of every page, oldest first. Pages are fetched one by one while you iterate.
+      #
+      # @example
+      #   client.changelog(count: 50).each { |event| puts event.id }
+      # @example Without a block, you get an Enumerator
+      #   client.changelog.each.first(5)
+      # @yieldparam event [Event] one changelog event. Events already seen in the overlap are skipped.
+      # @return [Changelog] self, when a block is given.
+      # @return [Enumerator<Event>] when no block is given.
+      # @raise [ApiError] on a non-2xx response.
+      # @raise [ConnectionError] on a network failure after all retries.
+      def each(&block)
+        return enum_for(:each) unless block
+
+        pages.each { |page| page.events.each(&block) }
+        self
+      end
+
+      # Lazy Enumerator of pages with already-seen events removed.
+      # Stops on the first page with no new events, or when the last event has no `processedAt`.
+      # A yielded Page has filtered `events` but the unfiltered `raw` response.
+      #
+      # @example Cursor of each page
+      #   client.changelog.pages.each { |page| puts "#{page.events.size} events, next #{page.next_start_time}" }
+      # @return [Enumerator<Changelog::Page>]
+      # @raise [ApiError] while iterating, on a non-2xx response.
+      # @raise [ConnectionError] while iterating, on a network failure after all retries.
+      def pages = Enumerator.new { |yielder| each_page(yielder) }
+
+      # Fetches one raw page. No overlap handling, so the cursor event comes back again.
+      #
+      # @param start_time [Integer, nil] cursor in epoch milliseconds. `nil` starts at the oldest event.
+      # @return [Changelog::Page]
+      # @raise [ApiError] on a non-2xx response.
+      # @raise [ConnectionError] on a network failure after all retries.
+      def page(start_time)
+        params = { q: "memberAndApplication", count: count, startTime: start_time }
+        Page.from_api(@connection.get(PATH, params))
+      end
+
+      private
+
+      def each_page(yielder)
+        Walk.new(since, []).run(yielder) { |start_time| page(start_time) }
+      end
+
+      def ensure_count
+        return if count.is_a?(Integer) && COUNT_RANGE.cover?(count)
+
+        raise ArgumentError, "count must be between #{COUNT_RANGE.min} and #{COUNT_RANGE.max}, got #{count.inspect}"
+      end
+    end
+  end
+end

data/lib/linkedin/member_data/cli/changelog_command.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module LinkedIn
+  module MemberData
+    class CLI
+      # Raw changelog events as a JSON array.
+      # @api private
+      class ChangelogCommand < Command
+        # @return [Array<Array>]
+        OPTIONS = [
+          ["--since DATE", SinceParser, "ISO date or datetime"],
+          ["--count N", Integer, "Events per request (1..50)"],
+          ["--out FILE", "Write to FILE instead of stdout"]
+        ].freeze
+
+        private
+
+        def execute
+          output.progress("Fetching changelog...")
+          events = client.changelog(**options.slice(:since, :count)).map(&:raw)
+          output.write(events, options[:out])
+          0
+        end
+      end
+    end
+  end
+end

data/lib/linkedin/member_data/cli/command.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module LinkedIn
+  module MemberData
+    class CLI
+      # Base for commands. A command parses its own options, then runs.
+      # `run` returns the exit code.
+      # @api private
+      class Command
+        # Option specs for OptionParser#on. Subclasses override.
+        # @return [Array<Array>]
+        OPTIONS = [].freeze
+        # Positional arguments the command accepts.
+        # @return [Integer]
+        MAX_ARGUMENTS = 0
+
+        # @param context [Context]
+        # @param argv [Array<String>] arguments after the command name. Options are removed from it.
+        def initialize(context, argv)
+          @context = context
+          @argv = argv
+          @options = {}
+        end
+
+        # Parses options, checks arguments and executes.
+        # @return [Integer] exit code.
+        # @raise [UsageError] on bad arguments.
+        def run
+          parser.parse!(argv, into: options)
+          context.use_token(options[:token]) if options.key?(:token)
+          check_arguments
+          execute
+        end
+
+        private
+
+        attr_reader :context, :argv, :options
+
+        def check_arguments
+          extra = argv.drop(self.class::MAX_ARGUMENTS)
+          raise UsageError, "unexpected argument: #{extra.first}" unless extra.empty?
+        end
+
+        def output = context.output
+
+        def client = context.client
+
+        def parser
+          parser = OptionParser.new("Usage: linkedin-member-data COMMAND [options]")
+          parser.on("--token TOKEN")
+          self.class::OPTIONS.each { |spec| parser.on(*spec) }
+          parser
+        end
+      end
+    end
+  end
+end

data/lib/linkedin/member_data/cli/global_options.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module LinkedIn
+  module MemberData
+    class CLI
+      # Options that come before the command name: --token and --help.
+      # @api private
+      class GlobalOptions
+        # @param argv [Array<String>] arguments. Global options are removed from it.
+        # @return [Hash{Symbol => Object}] options found, for example `{ token: "...", help: true }`.
+        # @raise [OptionParser::ParseError] on an unknown option.
+        def self.parse(argv) = {}.tap { |options| parser.order!(argv, into: options) }
+
+        def self.parser
+          OptionParser.new do |opts|
+            opts.on("--token TOKEN")
+            opts.on("-h", "--help")
+          end
+        end
+
+        private_class_method :parser
+      end
+    end
+  end
+end

data/lib/linkedin/member_data/cli/simple_commands.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module LinkedIn
+  module MemberData
+    class CLI
+      # Prints the gem version.
+      # @api private
+      class VersionCommand < Command
+        private
+
+        def execute
+          output.line(VERSION)
+          0
+        end
+      end
+
+      # Prints one snapshot domain per line. Needs no token.
+      # @api private
+      class DomainsCommand < Command
+        private
+
+        def execute
+          output.line(Domains::ALL)
+          0
+        end
+      end
+
+      # Prints the authorization of the token as JSON.
+      # @api private
+      class AuthCommand < Command
+        private
+
+        def execute
+          authorization = client.authorization
+          return missing if authorization.nil?
+
+          output.write(authorization.raw)
+          0
+        end
+
+        def missing
+          output.error("No authorization found for this token")
+          1
+        end
+      end
+    end
+  end
+end

data/lib/linkedin/member_data/cli/since_parser.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "time"
+require "date"
+require "optparse"
+
+module LinkedIn
+  module MemberData
+    class CLI
+      # Reads --since: an ISO datetime, or an ISO date (midnight UTC).
+      # @api private
+      class SinceParser
+        # @param value [String] ISO 8601 datetime or date.
+        # @return [Time]
+        # @raise [UsageError] when `value` is neither.
+        def self.call(value)
+          Time.iso8601(value)
+        rescue ArgumentError
+          from_date(value)
+        end
+
+        def self.from_date(value)
+          date = Date.iso8601(value)
+          Time.utc(date.year, date.month, date.day)
+        rescue ArgumentError
+          raise UsageError, "--since must be an ISO date or datetime, got #{value.inspect}"
+        end
+
+        private_class_method :from_date
+
+        # parser.on(..., SinceParser, ...) in ChangelogCommand relies on this.
+        OptionParser.accept(self) { |value| call(value) }
+      end
+    end
+  end
+end

data/lib/linkedin/member_data/cli/snapshot_command.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module LinkedIn
+  module MemberData
+    class CLI
+      # One domain to a file or stdout, or every domain into a directory.
+      # @api private
+      class SnapshotCommand < Command
+        # @return [Array<Array>]
+        OPTIONS = [
+          ["--all", "Download every domain"],
+          ["--out FILE", "Write to FILE instead of stdout"],
+          ["--out-dir DIR", "Directory for --all"]
+        ].freeze
+
+        # @return [Integer]
+        MAX_ARGUMENTS = 1
+
+        private
+
+        def check_arguments
+          super
+          options[:all] ? check_all : check_one
+        end
+
+        def check_all
+          raise UsageError, "--all cannot be used with --out" if options[:out]
+          raise UsageError, "--all takes no DOMAIN" unless argv.empty?
+        end
+
+        def check_one
+          raise UsageError, "--out-dir needs --all" if options[:"out-dir"]
+        end
+
+        def execute = options[:all] ? download_all : download_one
+
+        def download_one
+          domain = argv.shift
+          raise UsageError, "snapshot needs a DOMAIN (or --all)" if domain.nil?
+
+          output.write(fetch(domain.upcase), options[:out])
+          0
+        end
+
+        # A failing domain is reported and the run goes on. Exit 1 at the end.
+        def download_all
+          FileUtils.mkdir_p(out_dir)
+          failed = Domains::ALL.reject { |domain| saved?(domain, out_dir) }
+          failed.empty? ? 0 : 1
+        end
+
+        # The key comes from OptionParser's long option name (--out-dir).
+        def out_dir = options.fetch(:"out-dir") { raise UsageError, "--all needs --out-dir DIR" }
+
+        # A bad token fails every domain the same way, so Unauthorized and
+        # Forbidden stop the run at once. Other errors are per domain.
+        def saved?(domain, dir)
+          written?(domain, dir)
+        rescue Unauthorized, Forbidden
+          raise
+        rescue ApiError, ConnectionError => error
+          !reported?(domain, error)
+        end
+
+        def written?(domain, dir)
+          output.write(fetch(domain), File.join(dir, "#{domain}.json"))
+          true
+        end
+
+        # Reports the failure on stderr; true means the domain was not saved.
+        def reported?(domain, error)
+          output.error("#{domain}: #{error.message}")
+          true
+        end
+
+        def fetch(domain)
+          output.progress("Fetching #{domain}...")
+          client.snapshot(domain).to_a
+        end
+      end
+    end
+  end
+end

data/lib/linkedin/member_data/cli/support.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LinkedIn
+  module MemberData
+    class CLI
+      # Bad command line: unknown command, missing argument, bad value.
+      # @api private
+      class UsageError < StandardError; end
+
+      # Data goes to a file or stdout. Progress and errors go to stderr.
+      # @api private
+      class Output
+        # @param stdout [IO]
+        # @param stderr [IO]
+        def initialize(stdout, stderr)
+          @stdout = stdout
+          @stderr = stderr
+        end
+
+        # Files get the same bytes as stdout, including the final newline.
+        # @param data [Object] anything JSON can encode.
+        # @param path [String, nil] file to write. `nil` means stdout.
+        # @return [void]
+        def write(data, path = nil)
+          json = JSON.pretty_generate(data)
+          path ? File.write(path, "#{json}\n") : @stdout.puts(json)
+        end
+
+        # Prints a line to stdout.
+        # @param text [String, Array<String>]
+        # @return [void]
+        def line(text) = @stdout.puts(text)
+
+        # Prints a line to stderr.
+        # @param text [String]
+        # @return [void]
+        def error(text) = @stderr.puts(text)
+
+        alias progress error
+      end
+
+      # What every command needs: the output and a lazy client.
+      # The token is only looked up when a command asks for the client.
+      # @api private
+      class Context
+        # Name of the environment variable that holds the token.
+        # @return [String]
+        TOKEN_ENV = "LINKEDIN_ACCESS_TOKEN"
+
+        # @return [Output]
+        attr_reader :output
+
+        # @param output [Output]
+        # @param env [Hash, #[]] environment.
+        # @param client_factory [#call] takes a token and returns a `Client`.
+        # @param token [String, nil] token from `--token`.
+        def initialize(output:, env:, client_factory:, token:)
+          @output = output
+          @env = env
+          @client_factory = client_factory
+          @token = token
+        end
+
+        # The client, built on first use.
+        # @return [Client]
+        # @raise [ConfigurationError] when no token is set.
+        def client = @client ||= @client_factory.call(access_token)
+
+        # Replaces the token from `--token`.
+        # @param value [String]
+        # @return [String]
+        def use_token(value)
+          @token = value
+        end
+
+        private
+
+        def access_token
+          value = (@token || @env[TOKEN_ENV]).to_s.strip
+          raise ConfigurationError, "no access token: pass --token or set #{TOKEN_ENV}" if value.empty?
+
+          value
+        end
+      end
+    end
+  end
+end

data/lib/linkedin/member_data/cli.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+require_relative "../member_data"
+
+require_relative "cli/global_options"
+require_relative "cli/support"
+require_relative "cli/since_parser"
+require_relative "cli/command"
+require_relative "cli/simple_commands"
+require_relative "cli/snapshot_command"
+require_relative "cli/changelog_command"
+
+module LinkedIn
+  module MemberData
+    # Command line entry point. Data goes to stdout or files, progress and
+    # errors go to stderr. Exit codes: 0 ok, 1 API error, 2 usage error.
+    # @api private
+    class CLI
+      # Command name => command class.
+      # @return [Hash{String => Class}]
+      COMMANDS = {
+        "snapshot" => SnapshotCommand,
+        "changelog" => ChangelogCommand,
+        "domains" => DomainsCommand,
+        "auth" => AuthCommand,
+        "version" => VersionCommand
+      }.freeze
+
+      # Builds a `Client` from a token.
+      # @return [Proc]
+      DEFAULT_CLIENT_FACTORY = ->(token) { Client.new(access_token: token) }
+
+      # Help text.
+      # @return [String]
+      USAGE = <<~TEXT
+        Usage: linkedin-member-data COMMAND [options]
+
+        Commands:
+          snapshot DOMAIN [--out FILE]        Download one snapshot domain as a JSON array
+          snapshot --all --out-dir DIR        Download every domain, one <DOMAIN>.json per domain
+          changelog [--since DATE] [--count N] [--out FILE]
+                                              Download changelog events (last 28 days)
+          domains                             List snapshot domains
+          auth                                Show authorization status as JSON
+          version                             Print the gem version
+
+        Options:
+          --token TOKEN   Access token (or set LINKEDIN_ACCESS_TOKEN). Before or after the command.
+          -h, --help      Show this help
+      TEXT
+
+      # @param argv [Array<String>] command line arguments. Not changed.
+      # @param stdout [IO] where data goes.
+      # @param stderr [IO] where progress and errors go.
+      # @param env [Hash, #[]] environment, read for `LINKEDIN_ACCESS_TOKEN`.
+      # @param client_factory [#call] takes a token and returns a `Client`.
+      def initialize(argv, stdout: $stdout, stderr: $stderr, env: ENV, client_factory: DEFAULT_CLIENT_FACTORY)
+        @argv = argv.dup
+        @output = Output.new(stdout, stderr)
+        @env = env
+        @client_factory = client_factory
+      end
+
+      # Runs the command.
+      # @return [Integer] exit code: 0 ok, 1 API, network or file error, 2 usage error.
+      def run
+        handling_errors { start }
+      end
+
+      private
+
+      def start
+        options = GlobalOptions.parse(@argv)
+        options[:help] ? help : dispatch(options[:token])
+      end
+
+      def handling_errors
+        yield
+      rescue UsageError, OptionParser::ParseError, ConfigurationError, ArgumentError => error
+        usage_failure(error)
+      rescue Error, SystemCallError => error
+        failure(error)
+      end
+
+      def dispatch(token)
+        name = @argv.shift
+        raise UsageError, "missing command" if name.nil?
+
+        COMMANDS.fetch(name) { raise UsageError, "unknown command: #{name}" }.new(context(token), @argv).run
+      end
+
+      def context(token)
+        Context.new(output: @output, env: @env, client_factory: @client_factory, token: token)
+      end
+
+      def help
+        @output.line(USAGE)
+        0
+      end
+
+      def failure(error)
+        @output.error("error: #{error.message}")
+        1
+      end
+
+      def usage_failure(error)
+        @output.error("error: #{error.message}")
+        @output.error(USAGE)
+        2
+      end
+    end
+  end
+end