tinkit 0.0.0

data/lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb

@@ -0,0 +1,314 @@
+#require helper for cleaner require statements
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../../lib/helpers/require_helper')
+require Tinkit.helpers 'tk_escape'
+
+require 'aws/s3'
+
+module AWS::S3
+    class NoSuchBucket < ResponseError
+    end
+    
+    class BucketNotEmpty < ResponseError
+    end
+end
+
+module SdbS3Interface
+
+  class NilBucketError < StandardError
+  end
+  
+  class FilesMgr
+    include AWS
+    AccessKey = ENV["AMAZON_ACCESS_KEY_ID"]
+    SecretKey = ENV["AMAZON_SECRET_ACCESS_KEY"]
+
+    BucketNamespacePrefix = 'forforf'
+    
+    @@s3_connection = S3::Base.establish_connection!(:access_key_id => AccessKey,
+                                                                          :secret_access_key => SecretKey,
+                                                                          :persistent => false)
+    
+    attr_accessor :bucket_name
+
+    def initialize(glue_env, node_key_value)
+      #@s3_connection = S3::Base.establish_connection!(:access_key_id => AccessKey,
+      #                                                                    :secret_access_key => SecretKey)
+      @bucket_name = "#{BucketNamespacePrefix}_#{glue_env.user_datastore_location}"
+      #@attachment_bucket = use_bucket(@bucket_name)  #This can be stale!!!
+      #verify bucket is ready
+      #puts "Previous Response: #{S3::Service.response}"
+      #puts "#{__LINE__} - #{ S3::Service.buckets(true).map{|b| b.name} }"
+      #puts "This Bucket: #{@attachment_bucket.name}"
+      #puts "Last Response: #{S3::Service.response}"
+      #size = @attachment_bucket.size
+    end
+
+    #TODO: Move common file management functions from base node to here
+    def add(node, file_datas)
+      filenames = []
+      file_datas.each do |file_data|
+        filenames << file_data[:src_filename]
+      end
+
+      filenames.each do |filename|
+        basename = File.basename(filename)
+        esc_basename = TkEscape.escape(basename)
+        begin
+          S3::S3Object.store(esc_basename, open(filename), @bucket_name)
+        rescue AWS::S3::NoSuchBucket
+          puts "Rescued while adding files, retrying"
+          retry_request { S3::S3Object.store(esc_basename, open(filename), @bucket_name) }
+        end
+      end
+      #verify files are there
+      files = self.list_attachments
+    
+      filenames.each do |f|
+        bname = TkEscape.escape(File.basename(f))
+        esc_bname = TkEscape.escape(bname)
+        retry_request { S3::S3Object.store(esc_bname, open(f), @bucket_name) } unless files.include?(esc_bname)
+      end
+      
+      #update the metadata (have to wait until they're uploaded *sigh*
+      filenames.each do |f|
+        basename = File.basename(f)
+        attach_name = TkEscape.escape(basename)
+        begin
+          s3_obj = S3::S3Object.find(attach_name, @bucket_name)
+        rescue AWS::S3::NoSuchBucket
+          puts "Rescued while finding bucket, retrying"
+          retry_request { S3::S3Object.find(attach_name, @bucket_name) }
+        end  
+        modified_at = File.mtime(f).to_s
+        s3_obj.metadata[:modified_at] = modified_at
+        s3_obj.store
+      end
+      
+      filenames.map {|f| TkEscape.escape(File.basename(f))} #return basenames
+    end
+
+    def add_raw_data(node, file_name, content_type, raw_data, file_modified_at = nil)
+      
+      attach_name = TkEscape.escape(file_name)
+      
+      options = {:content_type => content_type}
+#=begin
+      begin
+        resp = S3::S3Object.store(attach_name, raw_data, @bucket_name, options)
+      rescue AWS::S3::NoSuchBucket
+        puts "Rescued while adding raw data, retrying"
+        retry_request { S3::S3Object.store(attach_name, raw_data, @bucket_name, options) }
+      end
+
+        obj = S3::S3Object.find(attach_name, @bucket_name)
+        obj.metadata[:modified_at] = "2011-01-13" #CGI.escape(file_modified_at)
+        obj.store
+
+      #verify files are there
+      max_wait_time = 20
+      now = Time.now
+      until (S3::S3Object.exists?(attach_name, @bucket_name) ) || (Time.now > (now + max_wait_time))
+        puts "Retrying store for add raw data"
+        sleep 2
+        S3::S3Object.store(attach_name, raw_data, @bucket_name, options)
+      end
+      
+      [attach_name]
+    end
+    
+    def list(node)
+      #conforming to base file mgr
+      list_attachments
+    end
+    
+    def subtract(node, file_basenames)
+      #conforming to base file mgr
+      subtract_files(node, file_basenames)
+    end
+    
+    def subtract_files(node, file_basenames)
+      if file_basenames == :all
+        subtract_all
+      else
+        subtract_some(file_basenames)
+      end
+    end
+
+    def get_raw_data(node, basename)
+      rtn = nil
+      
+      attach_name = TkEscape.escape(basename)
+
+      begin
+        rtn = S3::S3Object.value(attach_name, @bucket_name)
+      rescue AWS::S3::NoSuchBucket
+        puts "Rescued while getting raw data, bucket name: #{@bucket_name}"
+        begin
+          rtn = retry_request(attach_name, @bucket_name){|obj, buck| puts "sdbs3: #{obj.inspect} - #{buck.inspect}"; S3::S3Object.value(obj, buck)}
+        rescue AWS::S3::NoSuchKey
+          rtn = nil
+        end
+      rescue AWS::S3::NoSuchKey
+        rtn = nil
+      end
+      rtn
+    end
+
+    #todo change name to get_files_metadata
+    def get_attachments_metadata(node)
+      files_md = {}
+      begin
+        this_bucket = use_bucket(@bucket_name)
+        objects = this_bucket.objects
+      rescue  AWS::S3::NoSuchBucket
+        puts "rescued while getting objects from bucket to check metadata"
+        objects = retry_request{ this_bucket.objects }
+      end
+      objects.each do |object|
+        begin
+          obj_md = object.about.merge(object.metadata)
+        rescue AWS::S3::NoSuchBucket 
+          puts "Rescued while getting metadata from object"
+          obj_md = retry_request{object.about}
+        end
+        time_str = obj_md["x-amz-meta-modified-at"]||Time.parse(obj_md["last_modified"]).to_s
+        obj_md_file_modified = time_str
+        obj_md_content_type = obj_md["content-type"]
+        new_md = {:content_type => obj_md_content_type, :file_modified => obj_md_file_modified}
+
+        new_md.merge(obj_md)  #where does the original metadata go?
+        files_md[object.key] = new_md
+      end
+      files_md
+    end#def
+    
+    def list_objects
+      list = nil
+      this_bucket = use_bucket(@bucket_name)
+      begin
+        list = this_bucket.objects
+      rescue AWS::S3::NoSuchBucket
+        puts "Rescued while listing attachments"
+        list = retry_request{this_bucket.objects}
+      end
+      list
+    end
+    
+    def list_attachments
+      objs = list_objects
+      atts = objs.map{|o| o.key} if objs
+      atts || []
+    end
+    
+    def destroy_file_container
+      this_bucket = use_bucket(@bucket_name)
+      begin
+        this_bucket.delete(:force => true)
+      rescue AWS::S3::NoSuchBucket
+        puts "Running sanity check"
+        buckets = S3::Service.buckets(true).map{|b| b.name}
+        if buckets.include?(@bucket_name)
+          puts "AWS temporarily lost bucket before finding it so it can be deleted"
+          retry_request { this_bucket.delete(:force => true) }
+        end
+      end
+    end
+    
+    def subtract_some(file_basenames)
+      file_basenames.each do |basename|
+        attach_name = TkEscape.escape(basename)
+        S3::S3Object.delete(attach_name, @bucket_name)
+      end
+    end
+    
+    def subtract_all
+      #Changed behavior to leave bucket (this is different than other FileMgrs) 
+      this_bucket = use_bucket(@bucket_name)
+      begin  
+        this_bucket.delete_all
+      rescue AWS::S3::NoSuchBucket
+        puts "Bucket not found while deleting all. Maybe it's already been deleted?"
+        return nil
+        #aws_names = retry_request{@attachment_bucket.objects}
+      end
+      
+      max_wait_time = 20
+      now = Time.now
+      
+      while Time.now < (now + max_wait_time)
+        begin
+          this_bucket.delete
+        rescue AWS::S3::BucketNotEmpty
+          sleep 1
+          puts "Bucket not empty yet, trying again"
+          this_bucket.delete_all
+          next
+        end
+        break
+      end
+      
+      use_bucket(@bucket_name)
+      #file_basenames = aws_names.map{|o| o.key} if aws_names
+      #self.subtract_some(file_basenames) if file_basenames
+    end
+  
+    def retry_request(*args, &block)
+      puts "RETRYING Request with block: #{block.inspect}"
+      wait_time = 2
+      backoff_delay = 0.5
+      max_retries = 10
+      
+      resp = nil
+
+      1.upto(max_retries) do |i|
+        puts "Wating #{wait_time} secs to try again"
+        sleep wait_time
+        begin
+          resp = yield *args
+          raise TypeError, "Response was Nil, retrying" unless resp
+          break
+        rescue AWS::S3::NoSuchKey => e
+          raise e  #we want to raise this one"
+        rescue AWS::S3::ResponseError => e 
+          puts "rescued #{e.inspect}"
+          backoff_delay += backoff_delay# * i
+          wait_time += backoff_delay
+          if (wait_time > 3) && (e.class == AWS::S3::NoSuchBucket)
+            puts "Attempting to reset bucket"
+            @attachment_bucket = use_bucket(@bucket_name)
+          end
+          next
+        end#begin-rescue
+      end#upto
+      
+      
+    end#def
+    
+    def use_bucket(bucket_name)
+      begin
+        bucket = S3::Bucket.find(bucket_name)
+      rescue (AWS::S3::NoSuchBucket||NilBucketError) => e
+        begin
+          puts "Rescued error in use_bucket: #{e.inspect}"
+          S3::Bucket.create(bucket_name)
+          bucket = S3::Bucket.find(bucket_name)
+        rescue AWS::S3::NoSuchBucket #we just made it!!
+          bucket = retry_request(bucket_name){|buck_name| S3::Bucket.find(buck_name)}
+        end#inner begin-rescue
+      end#outer begin-rescue
+      
+      #verify bucket exists
+      found_buckets = S3::Service.buckets(true).map{|b| b.name}
+      unless found_buckets.include?(bucket_name)
+        #bucket = retry(:retry_block, bucket_name){|buck_name| S3::Bucket.find(buck_name)}
+      end#unless
+      unless bucket
+        puts "NIL Bucket cannot be returned"
+        retry_request(bucket_name){|buck_name| S3::Bucket.find(buck_name)}
+      end
+      raise(NilBucketError, "NIL Bucket cannot be returned",nil) unless bucket
+      return bucket
+    end#def
+    
+  end#class
+end#module

data/lib/glue_envs/sdb_s3_glue_env.rb

@@ -0,0 +1,248 @@
+#Tinkit directory structure defined in lib/helpers/require_helpers'
+require Tinkit.midas 'bufs_data_structure'
+require Tinkit.glue '/sdb_s3/sdb_s3_files_mgr'
+require Tinkit.helpers 'hash_helpers'
+require Tinkit.helpers 'log_helper'
+
+#require 'right_aws'
+require 'aws_sdb'  #published as forforf-aws-sdb
+#require 'aws/s3'
+require 'json'
+
+module SdbS3Env
+class GlueEnv
+  
+  
+  @@log = TinkitLog.set(self.name, :warn)
+   #used to identify metadata for models (should be consistent across models)
+  #PersistLayerKey not needed, node key can be used as persistent layer key
+  #see mysql_glue_env to decouple persistent layer key from node key
+  VersionKey = :_rev #to have timestamp
+  NamespaceKey = :sdbs3_namespace
+  
+  #MoabDataStoreDir = ".model"
+  #MoabDatastoreName = ".node_data.json"
+
+#TODO: Rather than using File class directly, should a special class be used? <- still applicable?
+attr_accessor :user_id,
+           :user_datastore_location,
+			     :metadata_keys,
+			     :required_instance_keys,
+			     :required_save_keys,
+			     :node_key,
+			     :model_key,
+			     :version_key,
+			     :namespace_key,
+			     :_files_mgr_class,
+           :views,
+			     :model_save_params,
+           :moab_data,
+           :persist_layer_key
+           #accessors specific to this persitence model
+           
+            
+            
+  def initialize(persist_env, data_model_bindings)
+    #TODO: determine if class_name is needed to segment cluster data within user data
+    #host = "https://sdb.amazonaws.com/"  (not provided by user)
+   
+    #user_id = env[:user_id]
+    sdb_s3_env = persist_env[:env]
+    #TODO: validations on format
+    domain_base_name = sdb_s3_env[:path]
+    @user_id = sdb_s3_env[:user_id]
+    @cluster_name = persist_env[:name]
+    
+    #data_model_bindings from NodeElementOperations
+    key_fields = data_model_bindings[:key_fields] 
+    initial_views_data = data_model_bindings[:views]
+    
+    @required_instance_keys = key_fields[:required_keys] #DataStructureModels::Tinkit::RequiredInstanceKeys
+    @required_save_keys = key_fields[:required_keys] #DataStructureModels::Tinkit::RequiredSaveKeys
+    @node_key = key_fields[:primary_key] #DataStructureModels::Tinkit::NodeKey
+    @persist_layer_key = @node_key
+    #@moab_datastore_name = MoabDatastoreName
+    @version_key = VersionKey  
+    @model_key = @node_key #ModelKey
+    @namespace_key = NamespaceKey
+    @metadata_keys = [@version_key, @namespace_key] 
+    aak = ENV["AMAZON_ACCESS_KEY_ID"]
+    asak = ENV["AMAZON_SECRET_ACCESS_KEY"]
+    #rightaws_log = TinkitLog.set("RightAWS::SDBInterface", :warn)
+    #sdb = RightAws::SdbInterface.new(aak, asak, :logger => rightaws_log, :multi_thread => true)    
+    sdb = AwsSdb::Service.new  #aws-sdb
+    @user_datastore_location = use_domain!(sdb, "#{domain_base_name}__#{@user_id}") 
+    @model_save_params = {:sdb => sdb, :domain => user_datastore_location, :node_key => @node_key}
+    @_files_mgr_class = SdbS3Interface::FilesMgr
+    @views = "temp"
+    @moab_data = {}
+    #@views_mgr = ViewsMgr.new({:data_file => @data_file_name})    
+    #@record_locker = {}  #tracks records that are in the process of being saved
+  end
+
+  def query_all  #TODO move to ViewsMgr
+    sdb = @model_save_params[:sdb]
+    domain = @model_save_params[:domain]
+    query = "select * from `#{domain}`"
+    raw_data = sdb.select(query).first
+    data = {}
+    #puts "QA Raw: #{raw_data.inspect}"
+    raw_data.each do |k,v|
+      data[k] = from_sdb(v)
+    end
+    @@log.info{"Query All data: #{data.values}"} if @@log.info?
+    data.values
+  end
+
+  #current relations supported:
+  # - :equals (data in the key field matches this_value)
+  # - :contains (this_value is contained in the key field data (same as equals for non-enumerable types )
+  def find_nodes_where(key, relation, this_value)
+    res = case relation
+      when :equals
+        find_equals(key, this_value)
+      when :contains
+        find_contains(key, this_value)
+    end #case
+    return res    
+  end
+  
+  def find_equals(key, this_value) 
+    results =[]
+    query_all.each do |record|
+      test_val = record[key]
+      results << record  if test_val == this_value
+    end
+    results  
+  end
+  
+  def find_contains(key, this_value) 
+    #there is probably an optimized way to do this natively
+    #in sdb's pseodo sql, but I can't figure it out
+    results =[]
+    query_all.each do |record|
+      test_val = record[key]
+      results << record  if find_contains_type_helper(test_val, this_value)
+    end
+    results  
+  end  
+
+  def find_contains_type_helper(stored_data, this_value)
+    #p stored_dataj
+    resp = nil
+    #stored_data = jparse(stored_dataj)
+    if stored_data.respond_to?(:"include?")
+      resp = (stored_data.include?(this_value))
+    else
+      resp = (stored_data == this_value)
+    end
+    return resp
+  end
+
+  def get(id)
+    sdb = @model_save_params[:sdb]
+    domain = @model_save_params[:domain]
+    raw_data = sdb.get_attributes(domain, id)
+    #puts "Raw Data: #{raw_data.inspect}"
+    data = from_sdb(raw_data)
+    data = nil if data.empty?
+    return data
+  end
+
+  def save(new_data)
+    sdb = @model_save_params[:sdb]
+    domain = @model_save_params[:domain]
+    #although we could pull @node_key directly, I do it this way to make it clear
+    #that it's a parameter used in saving to the persistence model
+    #I should try to be consistent on this
+    node_key = @model_save_params[:node_key]
+    rev_data = new_data.dup
+    rev_data[@version_key] = Time.now.hash
+    raw_model_data = HashKeys.sym_to_str(rev_data)
+    model_data = to_sdb(raw_model_data)
+    sdb.put_attributes(domain, new_data[node_key], model_data)
+    raw_model_data['rev'] = raw_model_data[@version_key]
+    return raw_model_data
+  end
+
+  def destroy_node(model_metadata)
+    sdb = @model_save_params[:sdb]
+    domain = @model_save_params[:domain]
+    #node_key = @model_save_params[:node_key]
+    item_name = model_metadata[@model_key]
+    @@log.info {"Deleting node: #{model_metadata.inspect} with key #{item_name} from domain: #{domain}"} if @@log.info?
+    sdb.delete_attributes(domain, item_name)
+  end
+  
+    #namespace is used to distinguish between unique
+    #data sets (i.e., users) within the model, for sdb, each user
+    #has their own domain, so the key only has to be unique within the domain
+  def generate_model_key(namespace, node_key)
+    "#{node_key}"
+    #"#{namespace}::#{node_key}"
+  end
+
+  def raw_all
+    query_all
+  end
+
+  def destroy_bulk(list_of_native_records)
+    sdb = @model_save_params[:sdb]
+    domain = @model_save_params[:domain]
+    node_key = @model_save_params[:node_key]
+    list_of_native_records.each do |rcd|
+      item_name = rcd[node_key]
+      #TODO: use the batch delete request
+      sdb.delete_attributes(domain, item_name)
+    end
+  end
+  
+  private
+  
+  def use_domain!(sdb, domain_name)
+    all_domains = parse_sdb_domains(sdb.list_domains)
+    if all_domains.include?(domain_name)
+      return domain_name
+    else #no domain by that name exists yet
+      sdb.create_domain(domain_name)
+      return domain_name
+    end
+  end
+    
+  def parse_sdb_domains(raw_list_results)
+    if raw_list_results.last == ""
+    #if raw_list_results[:next_token].nil? #right-aws
+      #return raw_list_results[:domains] #right-aws
+      return raw_list_results.first  #aws-sdb
+    else
+      raise "Have not implemented large list handling yet"
+    end
+  end
+
+  def from_sdb(sdb_data)
+    rtn_data = {}
+    sdb_data.each do |k_s, v_json|
+      k = k_s.to_sym
+      rtn_data[k] = jparse(v_json.first)
+    end
+    rtn_data
+  end
+
+  def to_sdb(data)
+    formatted_data = {}
+    data.each do |k,v|
+      k_f = k.to_s
+      v_f = v.to_json
+      formatted_data[k_f] = v_f
+    end
+    formatted_data
+  end
+  
+  def jparse(str)
+    return JSON.parse(str) if str =~ /\A\s*[{\[]/
+    JSON.parse("[#{str}]")[0]
+    #JSON.parse(str)
+  end
+  
+end#class
+end#module