smc-get 0.2.0.beta1 → 0.3.0.beta1

data/bin/smc-checksum~

@@ -0,0 +1,2 @@
+#!/usr/bin/env ruby
+#Encoding: UTF-8

data/bin/smc-get

@@ -1,31 +1,31 @@
-#!/usr/bin/env ruby
-#Encoding: UTF-8
-################################################################################
-# This file is part of smc-get.
-# Copyright (C) 2010-2011 Entertaining Software, Inc.
-# Copyright (C) 2011 Marvin Gülker
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-################################################################################
-
-if RUBY_VERSION =~ /^1.8/ #1.8 doesn't have require_relative
-  require File.join(File.expand_path(File.dirname(__FILE__)), "..", "lib", "smc_get", "smc_get")
-  require File.join(File.expand_path(File.dirname(__FILE__)), "..", "lib", "smc_get", "cui")
-else
-  require_relative "../lib/smc_get/smc_get"
-  require_relative "../lib/smc_get/cui"
-end
-
-cui = SmcGet::CUI.new(ARGV)
-cui.start
+#!/usr/bin/env ruby
+#Encoding: UTF-8
+################################################################################
+# This file is part of smc-get.
+# Copyright (C) 2010-2011 Entertaining Software, Inc.
+# Copyright (C) 2011 Marvin Gülker
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+if RUBY_VERSION =~ /^1.8/ #1.8 doesn't have require_relative
+  require File.join(File.expand_path(File.dirname(__FILE__)), "..", "lib", "smc_get", "smc_get")
+  require File.join(File.expand_path(File.dirname(__FILE__)), "..", "lib", "smc_get", "cui")
+else
+  require_relative "../lib/smc_get/smc_get"
+  require_relative "../lib/smc_get/cui"
+end
+
+cui = SmcGet::CUI.new(ARGV)
+cui.start

data/bin/smc-repo-conv

@@ -0,0 +1,205 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env ruby
+#Note the user of string hash keys throughout this file. This is
+#due to the package specifications being serialized using YAML,
+#which shouldn't be used with symbols. To prevent the necessity
+#of converting between symbols and strings everywhere a spec needs
+#to be accessed, I just use strings as the hash keys.
+
+require "open-uri"
+require "fileutils"
+require "pathname"
+require "tempfile"
+require "psych"
+require "yaml"
+require_relative "../lib/smc_get"
+
+PROGNAME = File.basename($0)
+
+if ARGV.empty? or ARGV.include?("-h") or ARGV.include?("--help")
+  puts(<<HELP)
+USAGE:
+  #{PROGNAME} OUTPUT_DIRECTORY
+
+Skims through the GitHub SMC repository that contains all source
+files for the SMCPAK files in the main repository and converts
+them all to SMCPAKs. A complete SMC repository is created inside
+OUTPUT_DIRECTORY.
+HELP
+exit
+end
+
+Thread.abort_on_exception = true
+
+OUTPUT_DIRECTORY   = Pathname.new(ARGV.pop)
+SPECS_DIRECTORY    = OUTPUT_DIRECTORY + "specs"
+PACKAGES_DIRECTORY = OUTPUT_DIRECTORY + "packages"
+REPO_LIST_FILE     = OUTPUT_DIRECTORY + "packages.lst"
+
+SOURCE_REPO_BASE_URL = "https://raw.github.com/Luiji/Secret-Maryo-Chronicles-Contributed-Levels/master"
+SPECS_BASE_URL       = SOURCE_REPO_BASE_URL + "/packages/"
+PACKAGE_LIST_URL     = SPECS_BASE_URL + "packages.lst"
+ELEMENT_BASE_URLS    = {
+  "levels"   => SOURCE_REPO_BASE_URL + "/levels/",
+  "graphics" => SOURCE_REPO_BASE_URL + "/graphics/",
+  "music"    => SOURCE_REPO_BASE_URL + "/music/",
+  "sounds"   => SOURCE_REPO_BASE_URL + "/sounds/",
+  "worlds"   => SOURCE_REPO_BASE_URL + "/worlds/"
+}
+
+ELEMENT_TEMP_DIRS = {
+  "levels"   => "levels",
+  "graphics" => "pixmaps",
+  "music"    => "music",
+  "sounds"   => "sounds",
+  "worlds"   => "worlds"
+}
+
+########################################
+# Start of code
+########################################
+
+@output_mutex = Mutex.new
+@threads = []
+Thread.abort_on_exception = true
+
+def info(str)
+  @output_mutex.synchronize do
+    puts(str) #TODO: Handling the --quiet option!
+  end
+end
+
+def error(str)
+  @output_mutex.synchronize do
+    $stderr.puts(str)
+  end
+end
+
+def make_spec(name, old_spec, directory)
+  new_spec = old_spec.dup
+  new_spec["checksums"] = Hash.new{|hsh, k| hsh[k] = Hash.new(&hsh.default_proc)} #Recursive hash
+  
+  #This is a dirty workaround for the fact that the GitHub specs don't obey
+  #the specification structure and use a field "last-updated" instead of
+  #the correct "last_update" (note the missing 'd' at the end).
+  if new_spec["last-updated"]
+    $stderr.puts("**WARNING** Invalid field `last-updated' autocorrected to `last_update' for #{name}!")
+    new_spec["last_update"] = new_spec.delete("last-updated")
+  end
+  
+  #First, go for the similar-structured spec elements
+  %w[levels graphics music sounds].each do |str|
+    next unless old_spec[str] #May be missing if not needed
+    
+    old_spec[str].each do |file|
+      file = Pathname.new(file)
+      #Recursively grab the second-to-last item (the last item is the
+      #file to check itself which we need to assign a value, the checksum,
+      #to). If a file’s parent is "." there’s no need to iterate through the
+      #hash as the file is _directly_ inside an element’s toplevel directory
+      #(where element is one of "levels", "graphics", etc.).
+      hsh = new_spec["checksums"][str]
+      file.parent.each_filename{|part| hsh = hsh[part]} unless file.parent.to_s == "."
+      
+      checksum = Digest::SHA1.file(directory + ELEMENT_TEMP_DIRS[str] + file).hexdigest
+      hsh[file.basename.to_s] = checksum
+    end
+  end
+
+  #Then go for the worlds.
+  if old_spec["worlds"] #May be missing if not needed
+    old_spec["worlds"].each do |worldname|
+      %w[description.xml layer.xml world.xml].each do |filename|
+        checksum = Digest::SHA1.file(directory + worldname + filename).hexdigest
+        new_spec["checksums"]["worlds"][worldname][filename] = checksum
+      end
+    end
+  end
+
+  File.open(directory + "#{name}.yml", "w"){|f| YAML.dump(new_spec, f)}
+end
+
+#include FileUtils::Verbose #We want to see what’s going on
+include FileUtils
+
+#Initialize smc-get
+SmcGet.setup
+
+#Setup the bare repository directory structure
+rm_rf OUTPUT_DIRECTORY
+mkdir_p OUTPUT_DIRECTORY
+mkdir PACKAGES_DIRECTORY
+mkdir SPECS_DIRECTORY
+
+#Get the list of all available packages
+info("Reading remote package list")
+package_names = open(PACKAGE_LIST_URL){|page| page.readlines.map(&:chomp)}
+skipped = []
+
+#Iterate through all the packages
+package_names.each do |pkgname|
+  #To speed up the whole thing, do each package in it’s own thread
+  @threads << Thread.new do
+    #Get the remote package spec
+    info("Reading remote package specification for '#{pkgname}'")
+    spec_url = SPECS_BASE_URL + "#{pkgname}.yml"
+    spec = open(spec_url){|page| YAML.load(page)}
+
+    #Create a temporary directory just for this sole package
+    Dir.mktmpdir do |tmpdir|
+      tmpdir = Pathname.new(tmpdir) + pkgname #Package directory must be named the same as the spec file
+      mkdir_p tmpdir
+      
+      begin
+        #Download all the package’s files into the appropriate directories
+        %w[levels graphics music sounds].each do |str|
+          next unless spec[str] #Not all specs contain everything
+          info("Downloading #{str} for '#{pkgname}'")
+          
+          spec[str].each do |file|
+            file = Pathname.new(file)
+            downloaded_file = tmpdir + ELEMENT_TEMP_DIRS[str] + file
+            mkdir_p downloaded_file.dirname #Subdirectories usually allowed
+            
+            open(ELEMENT_BASE_URLS[str] + URI.escape(file.to_s)) do |page|
+              File.open(downloaded_file, "w"){|f| f.write(page.read)}
+            end #open URI
+          end #each file
+        end #each string
+
+        info("Computing checksums for '#{pkgname}'")
+        make_spec(pkgname, spec, tmpdir)
+        info("Copying '#{pkgname}' to '#{OUTPUT_DIRECTORY}'")
+        pkg = SmcGet::Package.create(tmpdir)
+        cp pkg.path, PACKAGES_DIRECTORY
+        pkg.spec.save(SPECS_DIRECTORY)
+      rescue => e
+        skipped << pkgname
+        error("**WARNING** Skipping package '#{pkgname}', original error was: #{e.class}: #{e.message}")
+      end
+    end #mktmpdir
+    
+  end #Thread.new
+end #package_names.each
+
+#Wait for all the threads to finish
+@threads.map(&:join)
+
+#Write the package list
+converted_packages = package_names - skipped
+File.open(REPO_LIST_FILE, "w"){|f| f.write(converted_packages.join("\n"))}
+
+#Print out a summary... Hey, waiting makes curious! ;-)
+puts
+puts "SUMMARY".center(80, "=")
+puts "Packages found in the GitHub repository: #{package_names.count}"
+sleep 2
+puts "Packages converted into .smcpak files: #{converted_packages.count}"
+sleep 2
+puts "Packages skipped due to packaging errors: #{skipped.count}"
+sleep 2
+puts
+print "That is... "
+sleep 3
+percent = ((converted_packages.count.to_f / package_names.count.to_f) * 100).round(2)
+puts "#{percent}% converted correctly! Have fun!"

data/bin/smc-repo-conv~

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env ruby
+
+require "open-uri"
+require "fileutils"
+require "pathname"
+require "tempfile"
+require "psych"
+require "yaml"
+require_relative "../lib/smc_get"
+
+PROGNAME = File.basename($0)
+
+
+if ARGV.empty? or ARGV.include?("-h") or ARGV.include?("--help")
+  puts(<<HELP)
+USAGE:
+  #{PROGNAME} OUTPUT_DIRECTORY
+
+Skims through the GitHub SMC repository that contains all source
+files for the SMCPAK files in the main repository and converts
+them all to SMCPAKs. A complete SMC repository is created inside
+OUTPUT_DIRECTORY.
+HELP
+exit
+end
+
+OUTPUT_DIRECTORY   = Pathname.new(ARGV.pop)
+SPECS_DIRECTORY    = OUTPUT_DIRECTORY + "specs"
+PACKAGES_DIRECTORY = OUTPUT_DIRECTORY + "packages"
+
+SOURCE_REPO_BASE_URL = "https://raw.github.com/Luiji/Secret-Maryo-Chronicles-Contributed-Levels/master"
+SPECS_BASE_URL       = SOURCE_REPO_BASE_URL + "/packages/"
+PACKAGE_LIST_URL     = SPECS_BASE_URL + "packages.lst"
+ELEMENT_BASE_URLS    = {
+  :levels   => SOURCE_REPO_BASE_URL + "/levels/",
+  :graphics => SOURCE_REPO_BASE_URL + "/graphics/",
+  :music    => SOURCE_REPO_BASE_URL + "/music/",
+  :sounds   => SOURCE_REPO_BASE_URL + "/sounds/",
+  :worlds   => SOURCE_REPO_BASE_URL + "/worlds/"
+}
+
+ELEMENT_TEMP_DIRS = {
+  :levels   => "levels",
+  :graphics => "pixmaps",
+  :music    => "music",
+  :sounds   => "sounds",
+  :worlds   => "worlds"
+}
+
+########################################
+# Start of code
+########################################
+
+@output_mutex = Mutex.new
+@threads = []
+Thread.abort_on_exception = true
+
+def info(str)
+  @output_mutex.synchronize do
+    puts(str) #TODO: Handling the --quiet option!
+  end
+end
+
+def error(str)
+  @output_mutex.synchronize do
+    $stderr.puts(str)
+  end
+end
+
+def make_spec(name, old_spec, directory)
+  new_spec = old_spec.dup
+  new_spec["checksums"] = Hash.new{|hsh, k| hsh[k] = Hash.new(&hsh.default_proc)} #Recursive hash
+
+  %w[levels graphics music sounds].each do |str|
+    old_spec[str].each do |file|
+      file = Pathname.new(file)
+      hsh = new_spec["checksums"][str]
+      file.parent.each_filename{|part| hsh = hsh[part]} #Recursively grabbing second-to-last item
+      checksum = Digest::SHA1.file(directory + ELEMENT_TEMP_DIRS[str] + file).hexdigest
+      hsh[file.basename.to_s] = checksum
+    end
+  end
+
+  old_spec["worlds"].each do |worldname|
+    %w[description.xml layer.xml world.xml].each do |filename|
+      checksum = Digest::SHA1.file(directory + worldname + filename).hexdigest
+      new_spec["checksums"]["worlds"][worldname][filename] = checksum
+    end
+  end
+
+  File.open(directory + "#{name}.yml", "w"){|f| YAML.dump(new_spec, f)}
+end
+
+def makepkg(directory)
+
+end
+
+include FileUtils::Verbose #We want to see what’s going on
+
+rm_rf OUTPUT_DIRECTORY
+mkdir_p OUTPUT_DIRECTORY
+mkdir PACKAGES_DIRECTORY
+mkdir SPECS_DIRECTORY
+
+#Get the list of all available packages
+info("Reading remote package list")
+package_names = open(PACKAGE_LIST_URL){|page| page.readlines.map(&:chomp)}
+
+#Iterate through all the packages
+package_names.each do |pkgname|
+  #To speed up the whole thing, do each package in it’s own thread
+  @threads << Thread.new do
+    #Get the remote package spec
+    info("Reading remote package specification for '#{pkgname}'")
+    spec_url = SPECS_BASE_URL + "#{pkgname}.yml"
+    spec = open(spec_url){|page| YAML.load(page)}
+
+    #Create a temporary directory just for this sole package
+    Dir.mktmpdir do |tmpdir|
+      tmpdir = Pathname.new(tmpdir)
+      
+      begin
+        #Download all the package’s files into the appropriate directories
+        [:levels, :graphics, :music, :sounds].each do |sym|
+          next unless spec[sym.to_s] #Not all specs contain everything
+          info("Downloading #{sym} for '#{pkgname}'")
+          
+          spec[sym.to_s].each do |file|
+            file = Pathname.new(file)
+            downloaded_file = tmpdir + ELEMENT_TEMP_DIRS[sym] + file
+            mkdir_p downloaded_file.dirname #Subdirectories usually allowed
+            
+            open(ELEMENT_BASE_URLS[sym] + URI.escape(file.to_s)) do |page| 
+              File.open(downloaded_file, "w"){|f| f.write(page.read)}
+            end #open URI
+          end #each file
+        end #each symbol
+
+        info("Computing checksums for '#{pkgname}'")
+        make_spec(pkgname, spec, tmpdir)
+        info("Copying '#{pkgname}' to '#{OUTPUT_DIRECTORY}'")
+        pkg = SmcGet::Package.create(tmpdir)
+        cp pkg.path, PACKAGES_DIRECTORY
+        pkg.spec.save(SPECS_DIRECTORY)
+      rescue => e
+        error("Skipping package '#{pkgname}', original error was: #{e.message}")
+      end
+    end #mktmpdir
+    
+  end #Thread.new
+end #package_names.each
+info("MAIN PROGRAM FINISHED") #DEBUG
+@threads.map(&:join)
+sleep #DEBUG

data/config/smc-get.yml

@@ -1,32 +1,32 @@
-#Encoding: UTF-8
-################################################################################
-# This file is part of smc-get.
-# Copyright (C) 2010-2011 Entertaining Software, Inc.
-# Copyright (C) 2011 Marvin Gülker
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-################################################################################
-#
-#This is the configuration file for smc-get.
----
-#This is the directory where everything gets installed into.
-#Set this to the installation directory of SMC, e.g.
-#/usr/share/smc.
-data_directory: "/usr/share/smc"
-#This is the URL of the repository where packages are downloaded from.
-repo_url: "https://raw.github.com/Luiji/Secret-Maryo-Chronicles-Contributed-Levels/master/packages"
-#This is the maximum number of tries smc-get makes in order to 
-#download a package. 
-max_tries: 3
-
+#Encoding: UTF-8
+################################################################################
+# This file is part of smc-get.
+# Copyright (C) 2010-2011 Entertaining Software, Inc.
+# Copyright (C) 2011 Marvin Gülker
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+#
+#This is the configuration file for smc-get.
+---
+#This is the directory where everything gets installed into.
+#Set this to the installation directory of SMC, e.g.
+#/usr/share/smc.
+data_directory: "/usr/share/smc"
+#This is the URL of the repository where packages are downloaded from.
+repo_url: "https://raw.github.com/Quintus/smc-contributed-packages/master"
+#This is the maximum number of tries smc-get makes in order to 
+#download a package. 
+max_tries: 3
+