files.com 1.1.795 → 1.1.796

data/spec/models/file_transfer_spec.rb

@@ -2,9 +2,12 @@ require "spec_helper"
 require "stringio"
 require "tmpdir"
 require "zlib"
+require_relative "../support/stubbed_api"
 require_relative "../support/transfer_server"
 
 RSpec.describe Files::File, "transfers" do
+  include StubbedApi
+
   # 22,528 bytes that include every byte value, so text handling would show.
   let(:payload) { (0..255).map(&:chr).join.b * 88 }
   let(:signed_target) { "/objects/report.bin?X-Amz-Credential=fixture%2Fcredential&X-Amz-Signature=fixture-signature" }
@@ -53,6 +56,118 @@ RSpec.describe Files::File, "transfers" do
     complete ? "#{encoded}0\r\n\r\n" : encoded
   end
 
+  # The API and storage an upload sends through: the SDK's own connection with Faraday test stubs. Each begin_upload of
+  # report.bin grants the parts it asks for, partsize bytes each, at https://storage.example.test/parts/<number>; each
+  # part request is answered by part_response (200 with an ETag unless given), and the end of the upload with 201.
+  # requests keeps every request in order, with its parameters or body and its headers. A planning request past the
+  # tenth fails, so an upload that never ends stops.
+  def upload_api(requests, partsize:, part_response: nil)
+    stubbed_api_client do |stub|
+      stub.post("/api/rest/v1/file_actions/begin_upload/report.bin") do |env|
+        params = Faraday::Utils.parse_nested_query(env.body)
+        requests << [ :begin, params, env.request_headers ]
+        raise "The upload planned more than ten parts" if requests.count { |kind, *| kind == :begin } > 10
+
+        first = Integer(params.fetch("part"))
+        json_response(200, (first...(first + Integer(params.fetch("parts")))).map do |number|
+          { part_number: number, partsize: partsize, ref: "upload-ref", http_method: "PUT", upload_uri: "https://storage.example.test/parts/#{number}" }
+        end
+        )
+      end
+      stub.put(/\A\/parts\/\d+\z/) do |env|
+        number = Integer(env.url.path.split("/").last)
+        requests << [ :part, number, env.body, env.request_headers ]
+        part_response ? part_response.call(number) : [ 200, { "ETag" => "\"etag-#{number}\"" }, "" ]
+      end
+      stub.post("/api/rest/v1/files/report.bin") do |env|
+        requests << [ :end, Faraday::Utils.parse_nested_query(env.body), env.request_headers ]
+        json_response(201, { path: "report.bin", type: "file" })
+      end
+    end
+  end
+
+  def parts_sent(requests)
+    requests.select { |kind, *| kind == :part }.map { |_, number, body| [ number, body ] }
+  end
+
+  # A source that has only read, eof? and close: it gives its data at most max_read bytes a read, then, at its end,
+  # end_read (nil or an empty string). From stall_at on, every read gives an empty string although eof? is false. It
+  # refuses its hundred-and-first read, so a loop that never ends stops.
+  def short_reader(data, max_read:, end_read: nil, stall_at: nil)
+    offset = 0
+    reads = 0
+    Object.new.tap do |reader|
+      reader.define_singleton_method(:read) do |length = nil, _outbuf = nil|
+        raise "The source was read more than a hundred times" if (reads += 1) > 100
+        next "" if stall_at && offset >= stall_at
+        next end_read if offset >= data.bytesize
+
+        data.byteslice(offset, [ max_read, length || max_read ].min).tap { |chunk| offset += chunk.bytesize }
+      end
+      reader.define_singleton_method(:eof?) { offset >= data.bytesize }
+      reader.define_singleton_method(:close) { nil }
+    end
+  end
+
+  # Runs block with the path of a new local file holding data.
+  def with_local_file(data)
+    Dir.mktmpdir do |dir|
+      path = File.join(dir, "report.bin")
+      File.binwrite(path, data)
+      yield path
+    end
+  end
+
+  describe ".upload_chunks" do
+    it "fills each granted part from as many short reads as it takes" do
+      requests = []
+      described_class.upload_chunks(short_reader("abcdefghij", max_read: 3), "report.bin", { client: upload_api(requests, partsize: 4) })
+
+      expect(parts_sent(requests)).to eq([ [ 1, "abcd" ], [ 2, "efgh" ], [ 3, "ij" ] ])
+    end
+
+    it "ends a part at the source's end when its last read gives an empty string" do
+      requests = []
+      described_class.upload_chunks(short_reader("abcdefghij", max_read: 4, end_read: ""), "report.bin", { client: upload_api(requests, partsize: 4) })
+
+      expect(parts_sent(requests)).to eq([ [ 1, "abcd" ], [ 2, "efgh" ], [ 3, "ij" ] ])
+    end
+
+    it "refuses a source that gives nothing before its end, before sending that part" do
+      requests = []
+      reader = short_reader("abcdefghij", max_read: 4, stall_at: 4)
+
+      expect { described_class.upload_chunks(reader, "report.bin", { client: upload_api(requests, partsize: 4) }) }
+        .to raise_error(Files::InvalidParameterError, "The upload source returned no bytes before its end")
+      expect(parts_sent(requests)).to eq([ [ 1, "abcd" ] ])
+    end
+
+    it "lets a read's own error through, sending nothing after it" do
+      requests = []
+      read_error = IOError.new("the source failed")
+      reader = short_reader("abcdefghij", max_read: 4)
+      first_read = reader.method(:read)
+      reads = 0
+      reader.define_singleton_method(:read) { |*args| (reads += 1) == 2 ? raise(read_error) : first_read.call(*args) }
+
+      expect { described_class.upload_chunks(reader, "report.bin", { client: upload_api(requests, partsize: 4) }) }.to raise_error(IOError) { |error| expect(error).to equal(read_error) }
+      expect(parts_sent(requests)).to eq([ [ 1, "abcd" ] ])
+    end
+
+    it "reports each acknowledged part to progress, with an unknown total and this call's acknowledged bytes" do
+      requests = []
+      reports = []
+      described_class.upload_chunks(short_reader("abcdefghij", max_read: 3), "report.bin", { client: upload_api(requests, partsize: 4) }, progress: ->(report) { reports << report })
+
+      expect(reports).to eq([
+        { total_bytes: nil, acknowledged_bytes: 4, part_number: 1, part_bytes: 4 },
+        { total_bytes: nil, acknowledged_bytes: 8, part_number: 2, part_bytes: 4 },
+        { total_bytes: nil, acknowledged_bytes: 10, part_number: 3, part_bytes: 2 }
+      ]
+                           )
+    end
+  end
+
   describe ".upload_file" do
     it "raises the filesystem error when the local source does not exist" do
       Dir.mktmpdir do |dir|
@@ -61,6 +176,82 @@ RSpec.describe Files::File, "transfers" do
         expect { described_class.upload_file(missing_path, "report.bin") }.to raise_error(Errno::ENOENT)
       end
     end
+
+    it "uploads an empty file as one empty part" do
+      requests = []
+      with_local_file("") { |path| described_class.upload_file(path, "report.bin", { client: upload_api(requests, partsize: 4) }) }
+
+      expect(parts_sent(requests)).to eq([ [ 1, "" ] ])
+      expect(requests.last[0..1]).to match([ :end, hash_including("action" => "end", "size" => "0") ])
+    end
+
+    it "reports progress with the local file's size as its total, whatever size its planning parameters carry, and sends progress in no request" do
+      requests = []
+      reports = []
+      with_local_file("abcdefghij") do |path|
+        described_class.upload_file(path, "report.bin", { client: upload_api(requests, partsize: 4) }, params: { size: 999 }, progress: ->(report) { reports << report })
+      end
+
+      expect(requests.first[1]["size"]).to eq("999")
+      expect(reports).to eq([
+        { total_bytes: 10, acknowledged_bytes: 4, part_number: 1, part_bytes: 4 },
+        { total_bytes: 10, acknowledged_bytes: 8, part_number: 2, part_bytes: 4 },
+        { total_bytes: 10, acknowledged_bytes: 10, part_number: 3, part_bytes: 2 }
+      ]
+                           )
+      requests.each do |_kind, sent, *, headers|
+        expect(headers.keys.map(&:downcase)).not_to include("progress")
+        expect(sent.keys).not_to include("progress") if sent.is_a?(Hash)
+      end
+    end
+
+    it "refuses a part answered with a non-success status it returns, before reporting or ending the upload, when progress is given" do
+      requests = []
+      reports = []
+      client = upload_api(requests, partsize: 4, part_response: ->(_number) { [ 302, {}, "" ] })
+
+      with_local_file("abcdefghij") do |path|
+        expect { described_class.upload_file(path, "report.bin", { client: client }, progress: ->(report) { reports << report }) }
+          .to raise_error(Files::APIError, "Transfer request failed") { |error|
+            expect(error.http_status).to eq(302)
+            expect(error.full_message).not_to include("storage.example.test")
+          }
+      end
+      expect(reports).to be_empty
+      expect(requests.map(&:first)).to eq(%i[begin part])
+    end
+
+    it "uploads as before when a part's non-success status is returned and no progress is given" do
+      requests = []
+      client = upload_api(requests, partsize: 4, part_response: ->(_number) { [ 302, {}, "" ] })
+
+      with_local_file("abcdefghij") { |path| described_class.upload_file(path, "report.bin", { client: client }) }
+
+      expect(requests.map(&:first)).to eq(%i[begin part begin part begin part end])
+    end
+
+    it "stops at a part answered 500, reporting nothing and not ending the upload" do
+      requests = []
+      reports = []
+      client = upload_api(requests, partsize: 4, part_response: ->(_number) { [ 500, {}, "" ] })
+
+      with_local_file("abcdefghij") do |path|
+        expect { described_class.upload_file(path, "report.bin", { client: client }, progress: ->(report) { reports << report }) }.to raise_error(Files::APIConnectionError)
+      end
+      expect(reports).to be_empty
+      expect(requests.map(&:first)).to eq(%i[begin part])
+    end
+
+    it "ends the upload where its progress callback raises, with nothing planned, sent or ended after it, and closes its own file" do
+      requests = []
+      cancelled = StandardError.new("cancelled")
+      with_local_file("abcdefghij") do |path|
+        expect { described_class.upload_file(path, "report.bin", { client: upload_api(requests, partsize: 4) }, progress: ->(_report) { raise cancelled }) }
+          .to raise_error(StandardError) { |error| expect(error).to equal(cancelled) }
+        expect(ObjectSpace.each_object(::File).select { |file| !file.closed? && file.path == path }).to be_empty
+      end
+      expect(requests.map(&:first)).to eq(%i[begin part])
+    end
   end
 
   describe "#download_content" do

data/spec/models/file_underscore_destinations_spec.rb

@@ -16,7 +16,7 @@ RSpec.describe Files::File do
       "local/path/to/file.txt",
       "_/RemoteServers/42/file.txt",
       {},
-      params: {}
+      params: {}, progress: nil
     )
 
     described_class.upload_to_remote_server("local/path/to/file.txt", 42)

data/spec/spec_helper.rb

@@ -16,7 +16,15 @@ module MockServerFixture
   API_KEY = "synthetic-mock-server-api-key".freeze
   PARENT = Pathname.new("files_regional_worker")
 
+  # The shared API scenarios contract an example names with its :api_scenarios metadata, the definition's SHA-256.
+  API_SCENARIOS = { "format" => "files-api-scenarios", "format_version" => 1 }.freeze
+
   def self.origin
+    simulator.first
+  end
+
+  # The simulator's origin and the readiness report that showed it is the expected simulator.
+  def self.simulator
     values = %w[FILES_MOCK_SERVER_HOST FILES_MOCK_SERVER_PORT FILES_MOCK_SCHEMA_SHA256].to_h { |name| [ name, ENV[name].to_s.strip ] }
     missing = values.select { |_, value| value.empty? }.keys
     raise "Set #{missing.join(", ")} for the Files.com mock server's simulator; these examples never use Files.com." if missing.any?
@@ -26,11 +34,95 @@ module MockServerFixture
     raise "FILES_MOCK_SERVER_PORT must be a port number, not #{port.inspect}." unless port.match?(/\A\d{1,5}\z/) && port.to_i.between?(1, 65_535)
 
     origin = "http://#{host == "::1" ? "[::1]" : host}:#{port}"
+    report = ready(origin)
+    expect_simulator(origin, report, schema)
+    [ origin, report ]
+  end
+
+  def self.expect_simulator(origin, report, schema = ENV["FILES_MOCK_SCHEMA_SHA256"].to_s.strip)
     expected = { "status" => "ready", "mode" => "simulation", "contract_version" => 3, "schema_sha256" => schema }
-    reported = ready(origin).slice(*expected.keys)
+    reported = report.slice(*expected.keys)
     raise "#{origin} is not the expected simulator: it reports #{reported}, not #{expected}." unless reported == expected
+  end
+
+  # This run's hold on the shared daemon. @daemon is the origin and instance the run admitted first. @owned is the
+  # epoch this run made, { origin:, instance:, epoch:, mode:, identity: } (mode :default and identity nil after {}),
+  # set only once the reset's own answer and a fresh readiness report both showed it. @failed is the first reset that
+  # failed or could not be verified: once it is set, nothing resets the daemon again and the suite fails. The runtime
+  # must give the run a fresh daemon of its own: epoch 0 shows the boot profile, not that the daemon is empty or that
+  # nothing else uses it.
+  #
+  # Admits an example at the daemon `report` came from, before its journal baseline or any folder, and returns the
+  # contract its journal checks join, or nil. An example naming the API scenarios contract (definition_sha256) reuses
+  # an epoch this run made under that identity, or else resets the daemon into it, only on the instance first admitted
+  # and only from the boot epoch or a default epoch this run made. Any other example runs as before and sends nothing
+  # more, unless the daemon serves the contract: an epoch this run made is first returned to the default, and any
+  # other is refused. Nothing is reset for a refused example.
+  def self.admit(origin, report, definition_sha256)
+    # Once a reset failed or could not be verified, no example is admitted: nothing resets the daemon or makes folders.
+    raise_failed_transition
+    instance, epoch, active = report["instance"], report["epoch"], report.dig("profile", "api_scenarios")
+    @daemon ||= [ origin, instance ]
+    replaced = @daemon != [ origin, instance ]
+    @owned = nil if replaced || (@owned && [ @owned[:epoch], @owned[:identity] ] != [ epoch, active ])
 
-    origin
+    if definition_sha256
+      raise "#{origin} now answers as instance #{instance.inspect}, not #{@daemon.last.inspect}, the daemon this run admitted first; nothing was reset." if replaced
+
+      wanted = API_SCENARIOS.merge("definition_sha256" => definition_sha256)
+      return @owned if @owned && @owned[:identity] == wanted
+
+      raise "#{origin} is at epoch #{epoch.inspect} serving #{active.inspect}, which this run did not make; the API scenarios examples reset only the boot epoch or a default epoch this run made." unless (epoch == 0 && active.nil? && @owned.nil?) || (@owned && @owned[:mode] == :default)
+
+      transition(origin, instance, epoch, { "profile" => { "api_scenarios" => wanted } }, wanted)
+    elsif active
+      raise "#{origin} serves the API scenarios contract #{active.inspect} at epoch #{epoch.inspect}, which this run did not select; this example needs the default profile." unless @owned
+
+      transition(origin, instance, epoch, {}, nil)
+      nil
+    end
+  end
+
+  # Resets the daemon from `epoch` with `body` and verifies the next epoch on the same instance: first in the reset's
+  # own answer ({} is answered with the epoch alone; the contract adds the instance and its identity), then in a fresh
+  # readiness report. {} chooses the boot-default profile and an empty state; it restores nothing that came before. A
+  # reset attempted grants nothing: @owned is set only once both checks pass, and the first failure latches.
+  def self.transition(origin, instance, epoch, body, identity)
+    @owned = nil
+    response = reset(origin, body)
+    answer = JSON.parse(response.body)
+    expected = { "epoch" => epoch + 1 }.merge(identity ? { "instance" => instance, "api_scenarios" => identity } : {})
+    raise "the reset answered #{response.code} #{answer.inspect}, not 200 #{expected}" unless response.code == "200" && answer == expected
+
+    report = ready(origin)
+    expect_simulator(origin, report)
+    shown = [ report["instance"], report["epoch"], report.dig("profile", "api_scenarios") ]
+    raise "readiness after the reset shows instance, epoch and api_scenarios #{shown.inspect}, not #{[ instance, epoch + 1, identity ].inspect}" unless shown == [ instance, epoch + 1, identity ]
+
+    @owned = { origin: origin, instance: instance, epoch: epoch + 1, mode: identity ? :api_scenarios : :default, identity: identity }
+  rescue StandardError => e
+    @failed ||= "resetting #{origin} from epoch #{epoch.inspect} with #{JSON.generate(body)}: #{e.message}"
+    raise_failed_transition
+  end
+
+  def self.raise_failed_transition
+    raise "The shared daemon's API scenarios contract could not be entered or left (#{@failed}); this run resets it no further." if @failed
+  end
+
+  # After the suite: a failed transition fails the suite again, before anything else. Only an epoch this run made under
+  # the API scenarios contract is then returned to the default, once readiness still shows exactly that epoch; a run
+  # that made none sends nothing.
+  def self.finish
+    raise_failed_transition
+    return unless @owned && @owned[:mode] == :api_scenarios
+
+    origin, instance, epoch, identity = @owned.values_at(:origin, :instance, :epoch, :identity)
+    report = ready(origin)
+    expect_simulator(origin, report)
+    shown = [ report["instance"], report["epoch"], report.dig("profile", "api_scenarios") ]
+    raise "#{origin} now shows instance, epoch and api_scenarios #{shown.inspect}, not #{[ instance, epoch, identity ].inspect}, the epoch this run made; it is left as it is." unless shown == [ instance, epoch, identity ]
+
+    transition(origin, instance, epoch, {}, nil)
   end
 
   # Waits up to READY_TIMEOUT for something to listen at origin. Whatever answers first decides: a
@@ -57,18 +149,31 @@ module MockServerFixture
     Net::HTTP.start(uri.hostname, uri.port, open_timeout: 5, read_timeout: 10) { |http| http.get(uri.request_uri) }
   end
 
-  def self.journal(origin)
+  def self.reset(origin, body)
+    uri = URI("#{origin}/__files_mock/v1/reset")
+    Net::HTTP.start(uri.hostname, uri.port, open_timeout: 5, read_timeout: 10) { |http| http.post(uri.request_uri, JSON.generate(body), "Content-Type" => "application/json") }
+  end
+
+  # The journal's entries. Under the API scenarios contract the journal must also show the contract's origin, epoch,
+  # instance and identity, and that it dropped nothing, so its entries are the ones that epoch served.
+  def self.journal(origin, contract = nil)
     journal = JSON.parse(control(origin, "journal").body)
     raise "The simulator's journal is incomplete, so it cannot show which requests the SDK sent." unless journal["complete"] == true
 
+    if contract
+      shown = [ origin, *journal.values_at("epoch", "instance", "api_scenarios", "dropped") ]
+      expected = [ contract[:origin], contract[:epoch], contract[:instance], contract[:identity], 0 ]
+      raise "The simulator's journal shows origin, epoch, instance, api_scenarios and dropped #{shown.inspect}, not this example's #{expected.inspect}." unless shown == expected
+    end
+
     journal.fetch("entries")
   end
 
   # The API requests the example sent: each carried the same synthetic API key (the journal numbers keys
   # and never records them), no session, and the Ruby SDK's User-Agent. Transfers to the upload and
   # download URLs the simulator issued carry no credential, as presigned URLs do not.
-  def self.check_requests(origin, after)
-    entries = journal(origin).select { |entry| entry.fetch("seq") > after && entry.fetch("path").start_with?("/api/rest/v1/") }
+  def self.check_requests(origin, after, contract = nil)
+    entries = journal(origin, contract).select { |entry| entry.fetch("seq") > after && entry.fetch("path").start_with?("/api/rest/v1/") }
     raise "No API request reached the simulator." if entries.empty?
 
     keys = entries.map { |entry| entry.dig("credentials", "api_key") }.uniq
@@ -118,9 +223,10 @@ RSpec.configure do |config|
   config.around(:example, :with_test_folder) do |all|
     base_url = Files.base_url
     language = Files.language
-    origin = MockServerFixture.origin
+    origin, readiness = MockServerFixture.simulator
+    contract = MockServerFixture.admit(origin, readiness, all.metadata[:api_scenarios])
     Files.base_url = origin
-    after = MockServerFixture.journal(origin).map { |entry| entry.fetch("seq") }.max || 0
+    after = MockServerFixture.journal(origin, contract).map { |entry| entry.fetch("seq") }.max || 0
     @api_key = MockServerFixture::API_KEY
     @test_folder = MockServerFixture::PARENT.join(SecureRandom.uuid)
     sibling = MockServerFixture::PARENT.join("#{@test_folder.basename}-sibling")
@@ -135,7 +241,7 @@ RSpec.configure do |config|
     begin
       if created&.any?
         MockServerFixture.remove(created, @test_folder, sibling, { api_key: @api_key })
-        MockServerFixture.check_requests(origin, after)
+        MockServerFixture.check_requests(origin, after, contract)
       end
     ensure
       Files.base_url = base_url
@@ -145,6 +251,10 @@ RSpec.configure do |config|
     raise "The example left Files.language set to #{left_language.inspect}." unless left_language == language
   end
 
+  # A failed transition fails the suite before anything else; only an epoch this run made under the API scenarios
+  # contract is then returned to the default. A run that made none sends nothing.
+  config.after(:suite) { MockServerFixture.finish }
+
   config.shared_context_metadata_behavior = :apply_to_host_groups
   config.fail_if_no_examples = true
   config.disable_monkey_patching!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: files.com
 version: !ruby/object:Gem::Version
-  version: 1.1.795
+  version: 1.1.796
 platform: ruby
 authors:
 - files.com
@@ -483,6 +483,7 @@ files:
 - shared/path_comparison.json
 - shared/path_comparison.md
 - shared/url_test_data.json
+- spec/api_scenarios_spec.rb
 - spec/decimal_contract_spec.rb
 - spec/language_spec.rb
 - spec/lib/api_client_spec.rb