withruntime 0.1.0 → 0.3.0

data/lib/withruntime/sandbox.rb

@@ -13,22 +13,34 @@ module WithRuntime
     end
 
     # Creates a sandbox and, unless +wait: false+, waits until it is running.
-    # Every field is optional: with none you get the free trial while it lasts,
+    # Every field is optional: with none you get the free usage while it lasts,
     # the default region and a 2 vCPU / 4 GiB machine for up to 30 minutes.
-    # When every trial slot or the account's quota is taken, it waits for one to
+    # When every slot without credit or the account's quota is taken, it waits for one to
     # free, up to +wait_for_capacity+ seconds (the client's, two minutes by default).
     #
-    #   runtime.sandboxes.create(funding: "trial", labels: { team: "search" }, timeout_seconds: 900)
+    #   runtime.sandboxes.create(labels: { team: "search" }, timeout_seconds: 900)
+    #
+    # +spread_from: [id, ...]+ (one to 16 ids of your own sandboxes or snapshots)
+    # starts it on a server none of them is on. With no other server that has
+    # room, the create raises +no_capacity+ and nothing is made or charged.
+    # +sbx.info.placement+ is an opaque tag for its server: the same tag means
+    # the same server.
     def create(wait: true, idempotency_key: nil, wait_for_capacity: nil, **fields)
       room = wait_for_capacity.nil? ? @t.wait_for_capacity : [0, wait_for_capacity].max
       info = @t.json("POST", "/v1/sandboxes", body: Fields.body(fields), wait: wait ? 60 : 0,
                                               key: idempotency_key, room: room)
       sandbox = Sandbox.new(@t, info)
       if wait && sandbox.state != "running"
-        sandbox.wait_for("running", timeout: 60)
+        sandbox.wait_for("running", timeout: ApiDefaults::WAIT_FOR_TIMEOUT_SECONDS)
         unless sandbox.state == "running"
-          raise Error.new("Sandbox #{sandbox.id} is #{sandbox.state}, not running.", code: "start_failed",
-                                                                                       hint: "Read it with runtime.sandboxes.get(id); stop_reason says why.")
+          message = if info["replayed"]
+                      "Sandbox #{sandbox.id} was made by an earlier call with this idempotency key and is #{sandbox.state} now. " \
+                        "It did not fail to start; create again with a new key."
+                    else
+                      "Sandbox #{sandbox.id} is #{sandbox.state}, not running."
+                    end
+          raise Error.new(message, code: "start_failed",
+                                   hint: "Read it with runtime.sandboxes.get(id); stop_reason says why.")
         end
       end
       sandbox
@@ -94,7 +106,7 @@ module WithRuntime
 
     # Waits, on the server with no polling, until the sandbox reaches +state+
     # (running, paused or stopped) or +timeout+ seconds pass.
-    def wait_for(state, timeout: 60)
+    def wait_for(state, timeout: ApiDefaults::WAIT_FOR_TIMEOUT_SECONDS)
       seconds = [1, timeout.to_i].max
       @info = Record.new(@t.json("GET", path, query: { "waitFor" => state, "timeoutSeconds" => seconds },
                                               timeout: seconds + 60))
@@ -110,19 +122,28 @@ module WithRuntime
     # Saves the sandbox's memory and files; compute billing stops.
     def pause(wait: true, idempotency_key: nil) = lifecycle("pause", {}, wait, idempotency_key)
 
-    # Carries on a paused sandbox, with its memory and processes; +timeout_seconds+ is its new lease.
-    def wake(wait: true, timeout_seconds: nil, idempotency_key: nil)
-      lifecycle("wake", timeout_seconds ? { "timeoutSeconds" => timeout_seconds } : {}, wait, idempotency_key)
+    # Carries on a paused sandbox, with its memory and processes;
+    # +timeout_seconds+ is its new time limit, for a sandbox that has one.
+    # When its server has no room (+no_capacity+) it waits for room, as +create+
+    # does, up to +wait_for_capacity+ seconds (the client's, two minutes by
+    # default; 0 fails at once), sending the same idempotency key each time.
+    def wake(wait: true, timeout_seconds: nil, idempotency_key: nil, wait_for_capacity: nil)
+      lifecycle("wake", timeout_seconds ? { "timeoutSeconds" => timeout_seconds } : {}, wait, idempotency_key,
+                room: room_for(wait_for_capacity))
     end
 
-    # More time before the lease ends, at most an hour ahead of now.
+    # Moves a sandbox's time limit on, at most an hour ahead of now. A sandbox
+    # with no time limit answers at once and nothing changes.
     def extend_lease(seconds, idempotency_key: nil) = lifecycle("extend", { "seconds" => seconds }, false, idempotency_key)
 
     # Days (1 to 365) a paused sandbox is kept before it is deleted.
     def set_retention(days, idempotency_key: nil) = lifecycle("retention", { "days" => days }, false, idempotency_key)
 
     # Starts a stopped persistent sandbox again from its disk. Memory is not kept.
-    def restart(wait: true, idempotency_key: nil) = lifecycle("restart", {}, wait, idempotency_key)
+    # Waits for room like +wake+ when its server is full.
+    def restart(wait: true, idempotency_key: nil, wait_for_capacity: nil)
+      lifecycle("restart", {}, wait, idempotency_key, room: room_for(wait_for_capacity))
+    end
 
     # Changes name, labels, auto_wake, idle_pause_seconds, persistent or
     # max_total_cost_micros; fields left out stay as they are, and
@@ -133,12 +154,13 @@ module WithRuntime
       lifecycle("update", body, false, idempotency_key)
     end
 
-    # Keeps a running sandbox's lease ahead of now on a background thread, until
-    # +stop+ or +stop_keep_alive+: every +every+ seconds it extends the lease so
-    # that +margin+ seconds remain, never more than the hour ahead the API
-    # allows. Running time is billed as it is used. A paused sandbox is left
-    # paused; a stopped one ends the loop.
-    def keep_alive(every: 60, margin: 600, &on_error)
+    # Keeps a sandbox with a time limit running past it on a background thread,
+    # until +stop+ or +stop_keep_alive+: every +every+ seconds it extends the
+    # limit so that +margin+ seconds remain, never more than the hour ahead the
+    # API allows. A sandbox with no time limit needs none and is only watched.
+    # Running time is billed as it is used. A paused sandbox is left paused; a
+    # stopped one ends the loop.
+    def keep_alive(every: 60, margin: ApiDefaults::KEEP_ALIVE_MARGIN_SECONDS, &on_error)
       stop_keep_alive
       every = [10, every].max
       margin = margin.clamp(60, 3600)
@@ -148,7 +170,7 @@ module WithRuntime
             refresh
             break if %w[stopped stopping].include?(state)
 
-            expires = @info["expiresAt"]
+            expires = Sandbox.limit_end(@info)
             if state == "running" && expires
               need = (margin - (Time.parse(expires) - Time.now)).ceil
               extend_lease([3600, need].min) if need >= 1
@@ -163,6 +185,17 @@ module WithRuntime
       self
     end
 
+    # Where a sandbox's time limit ends, or nil when it has none: it renews
+    # itself (timeoutSeconds 0, or persistent) unless endsAt says its renewal
+    # stopped. An older server sends no endsAt and a timeout of at least a
+    # minute.
+    def self.limit_end(info)
+      return info["endsAt"] if info["endsAt"]
+      return nil if info["timeoutSeconds"].to_i.zero? || info["persistent"]
+
+      info["expiresAt"]
+    end
+
     # Ends a keep-alive, if one runs.
     def stop_keep_alive
       thread = @keep_alive
@@ -174,16 +207,19 @@ module WithRuntime
     # processes), each its own sandbox, on the same server, answered once they
     # run. One copy without +count+; an array with it. If a copy fails, the
     # error's +details["startedSandboxIds"]+ names the copies that did start.
+    # +funding+ is accepted and ignored: copies run on free usage first, then credit.
     def fork(count: nil, name: nil, labels: nil, keep_snapshot: nil, funding: nil, idempotency_key: nil)
-      body = Fields.body(count: count, name: name, labels: labels, keep_snapshot: keep_snapshot, funding: funding)
+      body = Fields.body(count: count, name: name, labels: labels, keep_snapshot: keep_snapshot)
       reply = @t.json("POST", path(":fork"), body: body, wait: 60, key: idempotency_key)
       copies = reply["sandboxes"].map { |info| Sandbox.new(@t, info) }
       count.nil? ? copies.first : copies
     end
 
     # Keeps this sandbox's whole machine as a snapshot to start new sandboxes
-    # from. A running sandbox is paused for the moment it takes, then woken; a
-    # paused one stays paused. A sandbox with volumes cannot be snapshotted.
+    # from. A running sandbox is paused by the server for the moment it takes
+    # and woken once the capture is over (one still capturing when the call
+    # answers stays paused until it is ready); a paused one stays paused. A
+    # sandbox with volumes cannot be snapshotted.
     def snapshot(name: nil, labels: nil, retention_days: nil, idempotency_key: nil)
       refresh
       # Straight after a fork or a wake the sandbox is still resuming, and
@@ -192,15 +228,11 @@ module WithRuntime
       if %w[resuming starting].include?(state) then wait_for("running")
       elsif state == "pausing" then wait_for("paused")
       end
-      running = state == "running"
-      pause if running
-      begin
-        Record.new(@t.json("POST", path(":snapshot"),
-                           body: Fields.body(name: name, labels: labels, retention_days: retention_days),
-                           wait: 10, key: idempotency_key))
-      ensure
-        wake if running
-      end
+      # The server pauses a running sandbox itself: a client pause and wake
+      # would be two more round trips.
+      Record.new(@t.json("POST", path(":snapshot"),
+                         body: Fields.body(name: name, labels: labels, retention_days: retention_days),
+                         wait: 10, key: idempotency_key))
     end
 
     # CPU and memory over a range: 15m, 1h, 6h, 24h, 7d or 30d.
@@ -239,43 +271,89 @@ module WithRuntime
                                                idempotency_key: idempotency_key)
       end
 
-      body = command_body(command, cwd, env, stdin, timeout || 86_400).merge("stream" => true)
+      longest = ApiDefaults::STREAMED_EXEC_TIMEOUT_SECONDS
+      body = command_body(command, cwd, env, stdin, timeout || longest).merge("stream" => true)
       handed_over = nil
-      @t.events("POST", path(":exec"), body: body, key: idempotency_key,
-                                       timeout: timeout ? timeout + 60 : 86_400) do |event|
-        case event["type"]
-        when "continue" then handed_over = event
-        when "error" then raise stream_error(event)
-        else yield event
+      process_id = nil
+      cursor = 0
+      exited = false
+      begin
+        @t.events("POST", path(":exec"), body: body, key: idempotency_key,
+                                         timeout: timeout ? timeout + 60 : longest) do |event|
+          case event["type"]
+          when "continue" then handed_over = event
+          when "error" then raise stream_error(event)
+          else
+            process_id = event["processId"] if event["type"] == "start"
+            cursor = event["offset"] + event["data"].bytesize if %w[stdout stderr].include?(event["type"])
+            exited = true if event["type"] == "exit"
+            yield event
+          end
+          break if handed_over || exited
         end
-        break if handed_over
+      rescue ConnectionError => e
+        raise if e.code != "connection_error" || process_id.nil?
+      end
+      return follow(handed_over["processId"], handed_over["cursor"], &block) if handed_over
+      return if exited
+
+      # The stream closed before the command's exit. Its output is kept on the
+      # sandbox, so follow it from where this stream stopped; an end with no
+      # exit is never a result.
+      if process_id.nil?
+        raise ConnectionError.new("The exec stream closed before the command started.",
+                                  code: "connection_error", idempotency_key: idempotency_key,
+                                  hint: "Run it again; to be sure it runs once, pass the same idempotency_key.")
       end
-      follow(handed_over["processId"], handed_over["cursor"], &block) if handed_over
+      follow(process_id, cursor, &block)
     end
 
-    # A process's output events from +cursor+ until it exits, across the server's stream slices.
+    # A process's output events from +cursor+ until it exits, across the
+    # server's stream slices. A stream that closes or is cut before the exit
+    # is followed again from where it stopped; four in a row with nothing new
+    # are an error, never an end.
     def follow(process_id, cursor = 0)
       return enum_for(:follow, process_id, cursor) unless block_given?
 
+      idle = 0
       loop do
         resume = nil
         exited = false
-        @t.events("GET", path("/processes/#{Transport.segment(process_id)}/output"),
-                  query: { "cursor" => cursor, "follow" => "true" }, timeout: 180) do |event|
-          case event["type"]
-          when "continue"
-            resume = event["cursor"]
-            break
-          when "stdout", "stderr" then cursor = event["offset"] + event["data"].bytesize
-          when "error" then raise stream_error(event)
-          when "exit" then exited = true
+        before = cursor
+        cut = false
+        begin
+          @t.events("GET", path("/processes/#{Transport.segment(process_id)}/output"),
+                    query: { "cursor" => cursor, "follow" => "true" }, timeout: 180) do |event|
+            case event["type"]
+            when "continue"
+              resume = event["cursor"]
+              break
+            when "stdout", "stderr" then cursor = event["offset"] + event["data"].bytesize
+            when "error" then raise stream_error(event)
+            when "exit" then exited = true
+            end
+            yield event
+            break if exited
           end
-          yield event
-          break if exited
+        rescue ConnectionError => e
+          raise if e.code != "connection_error"
+
+          cut = true
         end
-        break if exited || resume.nil?
+        break if exited
 
-        cursor = resume
+        cursor = [cursor, resume].max if resume
+        if resume || cursor > before
+          idle = 0
+          next
+        end
+        idle += 1
+        if idle > 3
+          raise ConnectionError.new("The output stream of process #{process_id} keeps closing before it ends.",
+                                    code: "connection_error",
+                                    hint: "Read what it printed so far: runtime sandboxes logs #{id} #{process_id}")
+        end
+        sleep([0.05 * (2**idle), 2].min) if cut
       end
     end
 
@@ -300,11 +378,13 @@ module WithRuntime
 
     private
 
-    def lifecycle(verb, body, wait, key)
-      @info = Record.new(@t.json("POST", path(":#{verb}"), body: body, wait: wait ? 60 : 0, key: key))
+    def lifecycle(verb, body, wait, key, room: 0)
+      @info = Record.new(@t.json("POST", path(":#{verb}"), body: body, wait: wait ? 60 : 0, key: key, room: room))
       self
     end
 
+    def room_for(wait_for_capacity) = wait_for_capacity.nil? ? @t.wait_for_capacity : [0, wait_for_capacity].max
+
     def command_body(command, cwd, env, stdin, timeout)
       body = command.is_a?(Array) ? { "argv" => command.map(&:to_s) } : { "command" => command.to_s }
       body["cwd"] = cwd if cwd
@@ -421,7 +501,7 @@ module WithRuntime
     end
 
     # A file's bytes (a binary String), any size.
-    def read(path) = t.bytes("GET", @sandbox.path("/files/content"), query: { "path" => path }, accept: "application/octet-stream")
+    def read(path) = t.file_bytes(@sandbox.path("/files/content"), query: { "path" => path })
 
     def read_text(path) = read(path).force_encoding(Encoding::UTF_8)
 
@@ -536,20 +616,38 @@ module WithRuntime
       nil
     end
 
-    # Copies a local file or directory in. A directory travels as one gzipped tar.
+    # Copies a local file or directory in. A directory travels as one gzipped
+    # tar to the API's folder routes, unpacked by the sandbox's own tar as it
+    # arrives.
     def upload(local, remote)
       return write(remote, File.binread(local)) unless File.directory?(local)
 
-      staging = "/tmp/.runtime-upload-#{SecureRandom.uuid}.tar.gz"
-      write(staging, Tar.pack_directory(local))
-      result = @sandbox.exec(["sh", "-c", 'mkdir -p "$1" && tar -xzf "$2" -C "$1"; code=$?; rm -f "$2"; exit $code',
-                              "sh", remote, staging])
-      raise CommandError, result unless result.ok?
-
+      archive = Tar.pack_directory(local)
+      if archive.bytesize <= CHUNK
+        t.bytes("PUT", @sandbox.path("/files/archive"), query: { "path" => remote }, raw: archive)
+        return nil
+      end
+      # Larger: parts in order; a part resent after a lost answer is not written twice.
+      upload = t.json("POST", @sandbox.path("/files/archive/uploads"), body: { "path" => remote, "gzip" => true })
+      base = @sandbox.path("/files/archive/uploads/#{Transport.segment(upload["uploadId"])}")
+      begin
+        (0...archive.bytesize).step(upload["chunkBytes"].to_i) do |offset|
+          t.bytes("PUT", base, query: { "offset" => offset }, raw: archive.byteslice(offset, upload["chunkBytes"].to_i))
+        end
+        t.json("POST", "#{base}:commit", body: {})
+      rescue StandardError
+        begin
+          t.json("POST", "#{base}:abort", body: {}, timeout: 10)
+        rescue Error
+          nil
+        end
+        raise
+      end
       nil
     end
 
-    # Copies a file or directory out.
+    # Copies a file or directory out. A directory comes as one tar from the
+    # API's folder route.
     def download(remote, local)
       entry = stat(remote)
       raise NotFoundError.new("#{remote} does not exist.", code: "file_not_found", status: 404) unless entry
@@ -559,18 +657,14 @@ module WithRuntime
         File.binwrite(local, read(remote))
         return nil
       end
-      staging = "/tmp/.runtime-download-#{SecureRandom.uuid}.tar.gz"
-      packed = @sandbox.exec(["tar", "-czf", staging, "-C", remote, "."])
-      raise CommandError, packed unless packed.ok?
-
+      # The sandbox's own tar packs it as it streams, and it is unpacked as it
+      # arrives; one cut short is refused and nothing is put in place.
+      unpacker = Tar::Unpacker.new(local)
       begin
-        Tar.unpack(read(staging), local)
+        t.chunks(@sandbox.path("/files/archive"), query: { "path" => remote, "gzip" => true }) { |chunk| unpacker.feed(chunk) }
+        unpacker.finish
       ensure
-        begin
-          remove(staging)
-        rescue Error
-          nil # /tmp is cleared with the sandbox.
-        end
+        unpacker.discard
       end
       nil
     end
@@ -587,11 +681,16 @@ module WithRuntime
       @sandbox = sandbox
     end
 
-    # Shares +port+, or changes its visibility if it is shared already.
-    # +visibility+ is "private" (the default: a token is needed) or "public";
-    # +ttl_seconds+ is how long the returned token lasts (60 s to 7 days).
-    def create(port, visibility: nil, ttl_seconds: nil)
-      body = { "port" => port, "visibility" => visibility, "ttlSeconds" => ttl_seconds }.compact
+    # Shares +port+, or changes its visibility and embed origins if it is
+    # shared already. +visibility+ is "private" (the default: a token is
+    # needed) or "public"; +ttl_seconds+ is how long the returned token lasts
+    # (60 s to 7 days). +embed_origins+ names the sites that may show it in an
+    # iframe (["https://app.example.com", "https://*.example.com"]); any other
+    # site's iframe is refused. Omitted, a shared port keeps its list; [] is
+    # any site, the default.
+    def create(port, visibility: nil, ttl_seconds: nil, embed_origins: nil)
+      body = { "port" => port, "visibility" => visibility, "ttlSeconds" => ttl_seconds,
+               "embedOrigins" => embed_origins }.compact
       Record.new(t.json("POST", path, body: body))
     end