an_post_return 0.2.0

data/lib/an_post_return/sftp/tracking_parser.rb

@@ -0,0 +1,116 @@
+require "csv"
+require "time"
+require_relative "../errors"
+
+module AnPostReturn
+  module SFTP
+    class TrackingParser
+      REQUIRED_FIELDS = %w[tracking_number status timestamp].freeze
+      TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S".freeze
+
+      # Parse tracking data from a CSV file
+      #
+      # @param file [File, Tempfile] CSV file to parse
+      # @return [Array<Hash>] Array of tracking data entries
+      def self.parse(file)
+        new.parse(file)
+      end
+
+      # Parse tracking data from a text file
+      #
+      # @param file_path [String] Path to the text file to parse
+      # @return [Hash] Hash containing header, data records, and footer information
+      # @raise [AnPostReturn::ParserError] if file is not found or empty
+      def parse(file_path)
+        validate_file!(file_path)
+
+        result = { header: nil, data: [], footer: nil }
+
+        File.foreach(file_path) do |line|
+          line = line.strip
+          next if line.empty?
+
+          # Determine delimiter and parse fields
+          delimiter = line.include?("+") ? "+" : ","
+          fields = CSV.parse_line(line, col_sep: delimiter, quote_char: '"')
+          record_type = fields[0]
+
+          case record_type
+          when "00"
+            result[:header] = parse_header(fields)
+          when "01"
+            result[:data] << parse_data_record(fields)
+          when "99"
+            result[:footer] = parse_footer(fields)
+          else
+            next # Skip unknown record types
+          end
+        end
+
+        result
+      rescue CSV::MalformedCSVError => e
+        raise ParserError, "Invalid file format: #{e.message}"
+      rescue => e
+        raise ParserError, "Error parsing tracking data: #{e.message}"
+      end
+
+      private
+
+      def validate_file!(file_path)
+        raise ParserError, "File not found: #{file_path}" unless File.exist?(file_path)
+        raise ParserError, "Empty file: #{file_path}" if File.zero?(file_path)
+      end
+
+      def parse_header(fields)
+        {
+          record_type: fields[0],
+          file_id: fields[1],
+          timestamp: format_timestamp(fields[2]),
+          record_count: fields[3].to_i,
+        }
+      end
+
+      def parse_data_record(fields)
+        {
+          record_type: fields[0],
+          service: fields[1],
+          tracking_number: fields[2],
+          country: fields[3],
+          combined_id: fields[4],
+          status: fields[5],
+          timestamp: format_timestamp(fields[6]),
+          location: fields[7],
+          notes: fields[8],
+          additional_info: fields[9],
+          outcome: fields[10],
+          recipient: fields[11],
+          extra1: fields[12],
+          extra2: fields[13],
+        }.compact
+      end
+
+      def parse_footer(fields)
+        { record_type: fields[0], record_count: fields[1].to_i }
+      end
+
+      def format_timestamp(timestamp)
+        return nil if timestamp.nil? || timestamp.empty?
+
+        if timestamp.length >= 14
+          year = timestamp[0..3]
+          month = timestamp[4..5]
+          day = timestamp[6..7]
+          hour = timestamp[8..9]
+          minute = timestamp[10..11]
+          second = timestamp[12..13]
+
+          Time.new(year.to_i, month.to_i, day.to_i, hour.to_i, minute.to_i, second.to_i)
+        else
+          Time.parse(timestamp)
+        end
+      rescue ArgumentError => e
+        nil # Return nil for invalid timestamps instead of raising error
+      end
+    end
+  end
+end

data/lib/an_post_return/tracker.rb

@@ -0,0 +1,97 @@
+require_relative "sftp/client"
+require_relative "sftp/tracking_parser"
+require_relative "configuration"
+
+module AnPostReturn
+  class Tracker
+    # Initialize a new Tracking resource
+    def initialize
+      @config = AnPostReturn.configuration
+      raise ArgumentError, "SFTP configuration is not set" unless @config.sftp_configured?
+    end
+
+    # Track files with a specific account number
+    #
+    # @param account_number [String] The account number to track
+    # @param last [Integer] The number of files to track
+    # @yieldparam data [Hash] Parsed tracking data containing :header, :data, and :footer
+    # @return [void]
+    # @raise [AnPostReturn::SFTP::ConnectionError] if SFTP connection fails
+    # @raise [AnPostReturn::SFTP::FileError] if file operations fail
+    def track_with_account_number(account_number, last: 0, &block)
+      # pad the account number with leading zeros to 8 digits
+      account_number = account_number.to_s.rjust(8, "0")
+      with_sftp_client do |sftp_client|
+        file =
+          if last.zero?
+            sftp_client.list_files(@config.sftp_config[:remote_path], "cdt#{account_number}*").first
+          else
+            sftp_client.list_files(@config.sftp_config[:remote_path], "cdt#{account_number}*")[-(last + 1)]
+          end
+        track_from(file.name, sftp_client, &block) if file
+      end
+    end
+
+    # Get tracking data from a file, incrementing file number if needed
+    #
+    # @param last_filename  [String, nil] Base last filename processed (e.g. "cdt0370132115864.txt")
+    # @param existing_sftp_client [SFTP::Client, nil] Existing SFTP client to use
+    # @yieldparam data [Hash] Parsed tracking data containing :header, :data, and :footer
+    # @return [void]
+    # @raise [AnPostReturn::SFTP::ConnectionError] if SFTP connection fails
+    # @raise [AnPostReturn::SFTP::FileError] if file operations fail
+    # @raise [AnPostReturn::ParserError] if parsing fails
+    def track_from(last_filename, existing_sftp_client = nil, &block)
+      return unless block_given?
+
+      with_sftp_client(existing_sftp_client) do |client|
+        # example file name: CDT99999999SSSSS.txt
+        # Where:
+        # •	CDT is to prefix each file (Customer Data Tracking).
+        # •	99999999 is the An Post Customer Account Number.
+        # •	SSSSS is a sequence number starting at 1 and incrementing by 1 for every file sent, with leading zeros.
+        # •	.txt is the standard file extension.
+        #
+        # extract the customer account number, sequence number and extension
+        customer_account_number = last_filename.match(/^cdt(\d+)([0-9]{5})\.txt$/)[1]
+        sequence_number = last_filename.match(/^cdt(\d+)([0-9]{5})\.txt$/)[2]
+        extension = ".txt"
+
+        while true
+          # increment the sequence number
+          sequence_number = sequence_number.to_i + 1
+          # format the new filename
+          next_filename = "cdt#{customer_account_number}#{sequence_number.to_s.rjust(5, "0")}#{extension}"
+
+          client.read_file(next_filename) do |tracking_file|
+            data = SFTP::TrackingParser.parse(tracking_file)
+            yield next_filename, data
+          end
+        end
+      end
+    rescue SFTP::FileNotFoundError
+      # If file not found, we're done
+      return
+    end
+
+    private
+
+    def with_sftp_client(existing_sftp_client = nil)
+      if existing_sftp_client
+        client = existing_sftp_client
+      else
+        client =
+          SFTP::Client.new(
+            @config.sftp_config[:host],
+            @config.sftp_config[:username],
+            @config.sftp_config[:password],
+            @config.proxy_config,
+          )
+        client.connect
+      end
+      yield client
+    ensure
+      client&.disconnect unless existing_sftp_client
+    end
+  end
+end

data/lib/an_post_return/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module AnPostReturn
+  VERSION = "0.2.0"
+end

data/lib/an_post_return.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+require "net/sftp"
+
+require_relative "an_post_return/version"
+require_relative "an_post_return/errors"
+
+module AnPostReturn
+  class << self
+    attr_writer :configuration
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+  end
+
+  # Autoload core classes
+  autoload :Configuration, "an_post_return/configuration"
+  autoload :Client, "an_post_return/client"
+
+  # Autoload object classes
+  autoload :Base, "an_post_return/objects/base"
+  autoload :ReturnLabel, "an_post_return/objects/return_label"
+
+  # Autoload tracker
+  autoload :Tracker, "an_post_return/tracker"
+
+  # Autoload resource classes
+  module Resources
+    autoload :Base, "an_post_return/resources/base"
+    autoload :ReturnLabelResource, "an_post_return/resources/return_label_resource"
+  end
+
+  # Autoload SFTP related classes
+  module SFTP
+    autoload :Client, "an_post_return/sftp/client"
+    autoload :Report, "an_post_return/sftp/report"
+  end
+end

data/scripts/example_prd.txt

@@ -0,0 +1,47 @@
+<context>
+# Overview  
+[Provide a high-level overview of your product here. Explain what problem it solves, who it's for, and why it's valuable.]
+
+# Core Features  
+[List and describe the main features of your product. For each feature, include:
+- What it does
+- Why it's important
+- How it works at a high level]
+
+# User Experience  
+[Describe the user journey and experience. Include:
+- User personas
+- Key user flows
+- UI/UX considerations]
+</context>
+<PRD>
+# Technical Architecture  
+[Outline the technical implementation details:
+- System components
+- Data models
+- APIs and integrations
+- Infrastructure requirements]
+
+# Development Roadmap  
+[Break down the development process into phases:
+- MVP requirements
+- Future enhancements
+- Do not think about timelines whatsoever -- all that matters is scope and detailing exactly what needs to be build in each phase so it can later be cut up into tasks]
+
+# Logical Dependency Chain
+[Define the logical order of development:
+- Which features need to be built first (foundation)
+- Getting as quickly as possible to something usable/visible front end that works
+- Properly pacing and scoping each feature so it is atomic but can also be built upon and improved as development approaches]
+
+# Risks and Mitigations  
+[Identify potential risks and how they'll be addressed:
+- Technical challenges
+- Figuring out the MVP that we can build upon
+- Resource constraints]
+
+# Appendix  
+[Include any additional information:
+- Research findings
+- Technical specifications]
+</PRD>

data/scripts/prd.txt

@@ -0,0 +1,147 @@
+# AnPost API Ruby Gem PRD
+
+## Overview
+The AnPost API Ruby gem provides a simple interface to interact with An Post's API services for return label creation and tracking information retrieval. The gem should support both production and test environments.
+
+## Core Requirements
+
+### Configuration
+- Simple configuration system with:
+  - test/production environment toggle
+  - HTTP proxy configuration (optional):
+    - proxy_host: Host of the proxy server
+    - proxy_port: Port number
+    - proxy_username: Optional proxy authentication username
+    - proxy_password: Optional proxy authentication password
+- Base URL determined by environment:
+  - Test: https://apim-anpost-mailslabels-nonprod.dev-anpost.com/returnsapi-q/v2/
+  - Production: https://apim-anpost-mailslabels.anpost.com/returnsapi/v2/
+
+### API Client
+- Clean, minimal HTTP client implementation using Faraday
+- Default JSON headers
+- Subscription key passed per-request, not in configuration
+- Proper error handling for API responses
+- Support for HTTP proxy configuration from global settings
+
+### Resources
+1. Return Label Resource
+   - Create return labels with subscription key authentication
+   - Handle all necessary parameters for label creation
+   - Support JSON request/response formats
+
+2. Tracking Resource
+   - Implement SFTP client based on existing implementation in sftp_test.rb:
+     - Support proxy configuration from global settings
+     - Connection management (connect/disconnect)
+     - File operations (read_file, list_files)
+     - Error handling for SFTP operations
+   
+   - Implement tracking parser based on existing implementation:
+     - Parse tracking file format (header, data records, footer)
+     - Support both '+' and ',' delimiters
+     - Handle quoted fields
+     - Structured data output with proper field mapping
+     
+   - Features:
+     - List available tracking files with glob pattern support
+     - Download and parse tracking files
+     - Stream processing for large files
+     - Proper cleanup of temporary files
+     - Support for incremental file processing
+     - Status tracking and aggregation
+
+## Technical Requirements
+- Ruby >= 3.0.0
+- Dependencies:
+  - faraday for HTTP requests
+  - net-sftp for SFTP operations
+  - net-ssh-http-proxy for SFTP proxy support
+  - json for data handling
+  - csv for tracking file parsing
+- Proper error classes and handling
+- Comprehensive test coverage
+- Standard Ruby code style
+
+## Implementation Notes
+- Keep configuration minimal and focused
+- Move authentication to request level
+- Support environment-based URL switching
+- Handle errors gracefully with descriptive messages
+- Follow Ruby gem best practices
+- Reuse existing SFTP and parsing code from sftp_test.rb
+- Ensure proper resource cleanup
+- Support proxy configuration across all components
+
+## SFTP Implementation Details
+1. SFTP Client Class:
+   ```ruby
+   module AnpostAPI
+     module SFTP
+       class Client
+         def initialize(host:, username:, password:, config: AnpostAPI.configuration)
+           # Initialize with credentials and use proxy from global config if available
+         end
+
+         def connect
+           # Establish SFTP connection with proxy support
+         end
+
+         def disconnect
+           # Clean disconnect
+         end
+
+         def read_file(remote_path, &block)
+           # Download and process file
+         end
+
+         def list_files(remote_path, glob_pattern = nil)
+           # List available files
+         end
+       end
+     end
+   end
+   ```
+
+2. Tracking Parser Class:
+   ```ruby
+   module AnpostAPI
+     module SFTP
+       class TrackingParser
+         def initialize(file_path)
+           # Initialize with file path
+         end
+
+         def parse
+           # Parse file contents
+           # Return structured data with header, records, and footer
+         end
+       end
+     end
+   end
+   ```
+
+3. Example Usage:
+   ```ruby
+   client = AnpostAPI::Client.new
+   
+   # Configure proxy if needed
+   AnpostAPI.configure do |config|
+     config.proxy_host = "proxy.example.com"
+     config.proxy_port = 3128
+     config.proxy_username = "user"
+     config.proxy_password = "pass"
+   end
+   
+   # Use tracking resource
+   client.tracking.get_updates(last_file: "cdt0379554008300.txt") do |filename, data|
+     # Process tracking data
+     puts "Processing file: #{filename}"
+     puts "Data: #{data}"
+   end
+   ```
+
+## Implementation Guidelines
+
+### 1. Code Structure
+```

data/scripts/sftp_test.rb

@@ -0,0 +1,185 @@
+require "net/sftp"
+require "pry"
+require "uri"
+require "net/ssh/proxy/http"
+require "x25519"
+require "tempfile"
+require "csv"
+
+class SFTPClient
+  def initialize(host, user, password, http_proxy_config = nil)
+    @host = host
+    @user = user
+    @password = password
+    @http_proxy_config = http_proxy_config
+  end
+
+  def connect
+    sftp_client.connect!
+  rescue Net::SSH::Exception => e
+    puts "Failed to connect to #{@host}"
+    puts e.message
+  end
+
+  def disconnect
+    sftp_client.close_channel
+    ssh_session.close
+  end
+
+  def read_file(remote_path, &block)
+    temp_file = Tempfile.new(["sftp", File.extname(remote_path)])
+    begin
+      @sftp_client.download!(remote_path, temp_file.path)
+      block_given? ? block.call(temp_file) : temp_file
+    rescue Net::SFTP::StatusException => e
+      puts "Failed to download #{remote_path}: #{e.message}"
+      nil
+    ensure
+      temp_file.close
+      temp_file.unlink
+    end
+  end
+
+  def list_files(remote_path, glob_pattern = nil)
+    if glob_pattern
+      @sftp_client.dir.glob(remote_path, glob_pattern).each { |entry| puts entry.longname }
+    else
+      @sftp_client.dir.foreach(remote_path) { |entry| puts entry.longname }
+    end
+  end
+
+  def sftp_client
+    @sftp_client ||= Net::SFTP::Session.new(ssh_session)
+  end
+
+  private
+
+  def ssh_session
+    ssh_options = { password: @password, auth_methods: ["password"] }
+
+    # Add proxy configuration if provided
+    if @http_proxy_config
+      ssh_options[:proxy] = Net::SSH::Proxy::HTTP.new(
+        @http_proxy_config[:host],
+        @http_proxy_config[:port],
+        user: @http_proxy_config[:username],
+        password: @http_proxy_config[:password],
+      )
+    end
+
+    @ssh_session ||= Net::SSH.start(@host, @user, ssh_options)
+  end
+end
+
+class TrackingDataParser
+  def initialize(file_path)
+    @file_path = file_path
+  end
+
+  def parse
+    result = { header: nil, data: [], footer: nil }
+
+    File.foreach(@file_path) do |line|
+      line = line.strip
+      next if line.empty?
+
+      # Determine delimiter and parse with CSV to handle quoted fields
+      delimiter = line.include?("+") ? "+" : ","
+      fields = CSV.parse_line(line, col_sep: delimiter, quote_char: '"')
+      record_type = fields[0]
+
+      case record_type
+      when "00"
+        result[:header] = parse_header(fields)
+      when "01"
+        result[:data] << parse_data_record(fields)
+      when "99"
+        result[:footer] = parse_footer(fields)
+      else
+        puts "Warning: Unknown record type: #{record_type}"
+      end
+    end
+
+    result
+  end
+
+  private
+
+  def parse_header(fields)
+    { record_type: fields[0], file_id: fields[1], timestamp: fields[2], record_count: fields[3].to_i }
+  end
+
+  def parse_data_record(fields)
+    {
+      record_type: fields[0],
+      service: fields[1],
+      tracking_number: fields[2],
+      country: fields[3],
+      combined_id: fields[4],
+      status: fields[5],
+      timestamp: fields[6],
+      location: fields[7],
+      notes: fields[8],
+      additional_info: fields[9],
+      outcome: fields[10],
+      recipient: fields[11],
+      extra1: fields[12],
+      extra2: fields[13],
+    }
+  end
+
+  def parse_footer(fields)
+    { record_type: fields[0], record_count: fields[1].to_i }
+  end
+
+  def format_timestamp(timestamp)
+    return nil if timestamp.nil?
+
+    if timestamp.length >= 14
+      year = timestamp[0..3]
+      month = timestamp[4..5]
+      day = timestamp[6..7]
+      hour = timestamp[8..9]
+      minute = timestamp[10..11]
+      second = timestamp[12..13]
+
+      "#{year}-#{month}-#{day} #{hour}:#{minute}:#{second}"
+    else
+      timestamp
+    end
+  end
+end
+
+# Example proxy configuration
+http_proxy_config = {
+  host: "sgp-forward-proxy.postco.co",
+  port: 3128,
+  username: "proxyuser", # Optional for HTTP
+  password: "VG17nL@qYgAFts", # Optional for HTTP
+}
+
+start_time = Time.now
+# With proxy
+sftp = SFTPClient.new("anpost.moveitcloud.eu", "ctuserpostco", "nUVG<akG[4@vc}^o", http_proxy_config)
+sftp.connect
+end_time = Time.now
+puts "Time taken to connect: #{end_time - start_time} seconds"
+
+# Rest of your code remains the same
+# sftp.list_files("/home/ctuserpostco", "cdt03795540083[4-9]*.txt")
+
+start = 379_554_008_300
+statuses = Set.new
+# increment by 1 for 10 times
+100.times do
+  file_name = "cdt0#{start += 1}.txt"
+  sftp.read_file("#{file_name}") do |temp_file|
+    parser = TrackingDataParser.new(temp_file.path)
+    data = parser.parse
+    data[:data].each { |record| statuses.add(record[:status]) }
+  end
+end
+
+puts "Statuses: #{statuses.size}"
+
+sftp.disconnect